{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "78814017",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.stats import truncnorm\n",
    "import gym\n",
    "import itertools\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import collections\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "d43f408e",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cpu'\n",
    "\n",
    "class Swish(nn.Module):\n",
    "    '''Swish激活函数'''\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return x * torch.sigmoid(x)\n",
    "\n",
    "def init_weights(m):\n",
    "    ''' 初始化模型权重 '''\n",
    "    def truncated_normal_init(t, mean=0.0, std=0.01):\n",
    "        torch.nn.init.normal_(t, mean=mean, std=std)\n",
    "        while True:\n",
    "            cond = (t < mean - 2 * std) | (t > mean + 2 * std)\n",
    "            if not torch.sum(cond):\n",
    "                break\n",
    "            t = torch.where(cond, torch.nn.init.normal_(torch.ones(t.shape, device=device), mean=mean, std=std), t)\n",
    "        return t\n",
    "    \n",
    "    if type(m) == nn.Linear or isinstance(m, FCLayer):\n",
    "        truncated_normal_init(m.weight, std=1 / (2 * np.sqrt(m._input_dim)))\n",
    "        m.bias.data.fill_(0.0)\n",
    "    \n",
    "class FCLayer(nn.Module):\n",
    "    ''' 集成之后的全连接层 '''\n",
    "    def __init__(self, input_dim, output_dim, ensemble_size, activeation):\n",
    "        super().__init__()\n",
    "        self._input_dim, self._output_dim = input_dim, output_dim\n",
    "        self.weight = nn.Parameter(torch.Tensor(ensemble_size, input_dim, output_dim).to(device))\n",
    "        self._activation = activeation\n",
    "        self.bias = nn.Parameter(torch.Tensor(ensemble_size, output_dim).to(device))\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self._activation(torch.add(torch.bmm(x, self.weight), self.bias[:, None, :]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f7d913d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 1, 3])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.arange(6).reshape(2,3)\n",
    "t[:, None, :].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f52dfb69",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EnsembleModel(nn.Module):\n",
    "    ''' 环境模型集成 '''\n",
    "    def __init__(self, state_dim, action_dim, ensemble_size=5, lr=1e-3):\n",
    "        ''' \n",
    "        类的初始化\n",
    "        核心功能\n",
    "            集成建模：包含ensemble_size个独立的神经网络（默认 5 个），每个网络学习环境动力学的一个估计。\n",
    "            输出设计：每个网络输出两部分：\n",
    "            均值（mean）：预测的状态变化量（下一状态 - 当前状态）和奖励。\n",
    "            对数方差（logvar）：预测不确定性的估计，通过约束确保方差在合理范围内。\n",
    "        关键参数\n",
    "            state_dim：状态空间维度。\n",
    "            action_dim：动作空间维度。\n",
    "            _output_dim：每个网络的输出维度，计算为 (state_dim + 1) * 2（+1为奖励，*2为均值和方差）。\n",
    "            _max_logvar和_min_logvar：限制对数方差的上下界，避免数值不稳定或不合理的方差估计。\n",
    "        网络结构\n",
    "            全连接层（FCLayer）：使用自定义的FCLayer类（可能为带集成的全连接层），每层包含ensemble_size个独立的子网络。\n",
    "            激活函数：前四层使用Swish激活函数（平滑非线性函数），最后一层使用恒等激活（输出线性值）。\n",
    "            参数初始化：通过init_weights函数初始化网络参数，提升训练稳定性。\n",
    "        '''\n",
    "        super().__init__()\n",
    "        # 输出包括均值和方差,因此是状态与奖励维度之和的两倍\n",
    "        self._output_dim = (state_dim + 1) * 2\n",
    "        '''\n",
    "        假设我们在训练一个机器人控制模型：\n",
    "        - 状态维度 state_dim = 3（如位置、速度等 3 个状态变量）\n",
    "        - 奖励维度 = 1（标量值，表示当前动作的好坏）\n",
    "        - 则输出维度为 (3 + 1) × 2 = 8\n",
    "        - 模型将输出：\n",
    "            - 3 个状态变量的均值\n",
    "            - 3 个状态变量的方差\n",
    "            - 1 个奖励值的均值\n",
    "            - 1 个奖励值的方差\n",
    "        '''\n",
    "\n",
    "        # 方差的上下界参数（可训练但固定初始值）\n",
    "        self._max_logvar = nn.Parameter((torch.ones((1, self._output_dim // 2)).float()/2).to(device), requires_grad= False)\n",
    "        '''self._max_logvar = Parameter containing: tensor([[0.5000, 0.5000, 0.5000, 0.5000]])'''\n",
    "        self._min_logvar = nn.Parameter((-torch.ones((1, self._output_dim // 2)).float()*10).to(device), requires_grad = False)\n",
    "        '''self._min_logvar = Parameter containing:tensor([[-10., -10., -10., -10.]])'''\n",
    "\n",
    "        # 五层全连接网络（使用FCLayer支持集成训练）\n",
    "        self.layer1 = FCLayer(state_dim + action_dim, 200, ensemble_size, Swish())\n",
    "        self.layer2 = FCLayer(200, 200, ensemble_size, Swish())\n",
    "        self.layer3 = FCLayer(200, 200, ensemble_size, Swish())\n",
    "        self.layer4 = FCLayer(200, 200, ensemble_size, Swish())\n",
    "        # 在 PyTorch 中，nn.Identity()是一个占位符模块，它返回输入不做任何修改。\n",
    "        self.layer5 = FCLayer(200, self._output_dim, ensemble_size, nn.Identity())\n",
    "        \n",
    "        # 初始化权重并设置优化器\n",
    "        self.apply(init_weights)\n",
    "        self.optimizer = torch.optim.Adam(self.parameters(), lr=lr)\n",
    "    \n",
    "    def forward(self, x, return_log_var=False):\n",
    "        '''\n",
    "        前向传播\n",
    "        流程解析\n",
    "            输入处理：输入x为状态与动作的拼接张量，形状为 [ensemble_size, batch_size, state_dim+action_dim]。\n",
    "            特征提取：通过五层全连接网络提取特征，最终输出ret形状为 [ensemble_size, batch_size, _output_dim]。\n",
    "        拆分均值和对数方差：\n",
    "            mean：前_output_dim//2维度为预测的均值（状态变化和奖励）。\n",
    "            logvar：通过softplus函数将网络输出约束到[_min_logvar, _max_logvar]区间，避免方差爆炸或消失。\n",
    "        输出格式：根据return_log_var参数，可选返回对数方差或方差（指数变换后）。\n",
    "        '''\n",
    "        # 前向传播计算预测结果\n",
    "        ret = self.layer5(self.layer4(self.layer3(self.layer1(x))))\n",
    "        # 提取均值\n",
    "        mean = ret[:, :, :self._output_dim // 2]\n",
    "\n",
    "        # 在PETS算法中,将方差控制在最小值和最大值之间\n",
    "        logvar = self._max_logvar - F.softplus(self._max_logvar -ret[:, :, self._output_dim // 2 :])\n",
    "        logvar = self._min_logvar + F.softplus(logvar - self._min_logvar)\n",
    "        return mean, logvar if return_log_var else torch.exp(logvar)\n",
    "    \n",
    "    def loss(self, mean, logvar, labels, use_var_loss=True):\n",
    "        '''\n",
    "        损失设计\n",
    "        带方差的损失（训练时）：\n",
    "            加权 MSE 损失：使用逆方差对预测误差加权，使模型更关注不确定性高的区域。\n",
    "            方差正则项：惩罚过大的对数方差，鼓励模型输出合理的不确定性估计。\n",
    "        仅 MSE 损失（验证时）：忽略方差，仅评估预测均值的准确性（如FakeEnv中的验证逻辑）。\n",
    "        '''\n",
    "        # 计算损失函数\n",
    "        inverse_var = torch.exp(-logvar)\n",
    "        if use_var_loss:\n",
    "            # 带方差正则化的损失\n",
    "            mse_loss = torch.mean(torch.mean(torch.pow(mean - labels, 2) * inverse_var, dim=-1), dim=-1)\n",
    "            var_loss = torch.mean(torch.mean(logvar, dim=-1), dim=-1)\n",
    "            total_loss = torch.sum(mse_loss) + torch.sum(var_loss)\n",
    "        else:\n",
    "            mse_loss = torch.mean(torch.pow(mean - labels, 2), dim=(1, 2))\n",
    "            total_loss = torch.sum(mse_loss)\n",
    "        return total_loss, mse_loss\n",
    "    \n",
    "    def train(self, loss):\n",
    "        '''\n",
    "        训练方法\n",
    "        优化逻辑\n",
    "            梯度清零：在反向传播前清空优化器梯度。\n",
    "            额外正则项：对_max_logvar和_min_logvar添加轻微正则化（代码中为固定值，可能用于平衡方差约束）。\n",
    "            参数更新：使用 Adam 优化器更新网络参数，最小化损失函数。\n",
    "        '''\n",
    "        # 训练步骤\n",
    "        self.optimizer.zero_grad()\n",
    "        # 添加对max_logvar和min_logvar的正则化\n",
    "        loss += 0.01 * torch.sum(self._max_logvar) - 0.01 * torch.sum(self._min_logvar)\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "2d62d7ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EnsembleDynamicsModel:\n",
    "    ''' 环境模型集成,加入精细化的训练 '''\n",
    "\n",
    "    def __init__(self, state_dim, action_dim, num_network=5):\n",
    "        '''\n",
    "        类的定义和初始化\n",
    "        类功能：集成多个动力学模型，提高预测的准确性和鲁棒性。\n",
    "        初始化参数：\n",
    "            state_dim：状态维度\n",
    "            action_dim：动作维度\n",
    "            num_network：集成模型的数量（默认 5 个）\n",
    "        核心组件：\n",
    "            EnsembleModel：底层的集成模型，包含多个网络\n",
    "            _epoch_since_last_update：记录自上次模型更新以来的训练轮数\n",
    "        '''\n",
    "        # 集成中模型的数量\n",
    "        self._num_network = num_network\n",
    "        self._state_dim, self._action_dim = state_dim, action_dim\n",
    "        # 基础模型\n",
    "        self.model = EnsembleModel(state_dim, action_dim, ensemble_size=num_network)\n",
    "        # 记录自上次模型更新以来的轮数\n",
    "        self._epoch_since_last_update = 0\n",
    "\n",
    "    def train(self, inputs, labels, batch_size=64, holdout_ratio=0.1, max_iter=20):\n",
    "        '''\n",
    "        模型训练方法\n",
    "        数据预处理：\n",
    "            将数据按比例划分为训练集和验证集\n",
    "            对验证集数据进行复制，使其可同时输入到所有网络\n",
    "        训练流程：\n",
    "            初始化每个网络的最佳模型存储\n",
    "            多轮训练：每轮为每个网络生成独立的数据排列\n",
    "            分批训练：每个批次的数据独立输入到每个网络\n",
    "            模型更新：计算预测分布的均值和方差，使用自定义损失函数优化\n",
    "        验证与早停：\n",
    "            每轮训练后在验证集上评估\n",
    "            调用_save_best方法保存表现最好的模型\n",
    "            满足早停条件（连续 5 轮无显著改进或达到最大轮数）时停止训练\n",
    "        '''\n",
    "        # 设置训练集与验证集\n",
    "        # 数据划分：训练集和验证集\n",
    "        permutation = np.random.permutation(inputs.shape[0])\n",
    "        inputs, labels = inputs[permutation], labels[permutation]\n",
    "        num_holdout = int(inputs.shape[0] * holdout_ratio)\n",
    "        train_inputs, train_labels = inputs[num_holdout:], labels[num_holdout:]\n",
    "        holdout_inputs, holdout_labels = inputs[:num_holdout], labels[:num_holdout]\n",
    "\n",
    "        # 转换为PyTorch张量并复制到所有集成模型\n",
    "        holdout_inputs = torch.from_numpy(holdout_inputs).float().to(device)\n",
    "        holdout_labels = torch.from_numpy(holdout_labels).float().to(device)\n",
    "        holdout_inputs = holdout_inputs[None, :, :].repeat([self._num_network, 1, 1])\n",
    "        holdout_labels = holdout_labels[None, :, :].repeat([self._num_network, 1, 1])\n",
    "\n",
    "        # 为每个模型保存最佳验证结果\n",
    "        self._snapshots = {i: (None, 1e10) for i in range(self._num_network)}\n",
    "\n",
    "        for epoch in itertools.count():\n",
    "            # 定义每一个网络的训练数据\n",
    "            # 为每个模型创建随机训练数据索引\n",
    "            train_index = np.vstack(\n",
    "                [np.random.permutation(train_inputs.shape[0]) for _ in range(self._num_network)]\n",
    "            )\n",
    "            # 所有真实数据都用来训练\n",
    "            for batch_start_pos in range(0, train_inputs.shape[0], batch_size):\n",
    "                batch_index = train_index[:, batch_start_pos:batch_start_pos + batch_size]\n",
    "                train_input = torch.from_numpy(train_inputs[batch_index]).float().to(device)\n",
    "                train_label = torch.from_numpy(train_labels[batch_index]).float().to(device)\n",
    "\n",
    "                 # 前向传播和反向传播\n",
    "                mean, logvar = self.model(train_input, return_log_var=True)\n",
    "                loss, _ = self.model.loss(mean, logvar, train_label)\n",
    "                self.model.train(loss)\n",
    "            \n",
    "            # 验证集评估\n",
    "            with torch.no_grad():\n",
    "                mean, logvar = self.model(holdout_inputs, return_log_var=True)\n",
    "                _, holdout_losses = self.model.loss(mean, logvar, holdout_labels, use_var_loss=False)\n",
    "                holdout_losses = holdout_losses.cpu()\n",
    "                break_condition = self._save_best(epoch, holdout_losses)\n",
    "                if break_condition or epoch > max_iter:\n",
    "                    break\n",
    "\n",
    "    def _save_best(self, epoch, losses, threshold=0.1):\n",
    "        '''\n",
    "        最佳模型保存方法\n",
    "        功能：监控每个网络在验证集上的性能，保存表现最佳的模型参数\n",
    "        判断逻辑：\n",
    "            计算当前损失与历史最佳损失的改进比例\n",
    "            若改进超过阈值（10%），则更新最佳模型记录\n",
    "            跟踪未更新的轮数，超过 5 轮则触发早停条件\n",
    "        '''\n",
    "        # 保存验证性能最佳的模型参数\n",
    "        updated = False\n",
    "        for i in range(len(losses)):\n",
    "            current = losses[i]\n",
    "            _, best = self._snapshots[i]\n",
    "            improvement = (best - current) / best\n",
    "            if improvement > threshold:\n",
    "                # 当验证损失有显著改善时保存模型\n",
    "                self._snapshots[i] = (epoch, current)\n",
    "                updated = True\n",
    "        self._epoch_since_last_update = 0 if updated else self._epoch_since_last_update + 1\n",
    "        # 如果长时间没有改善则停止训练\n",
    "        return self._epoch_since_last_update > 5\n",
    "\n",
    "    def predict(self, inputs, batch_size=64):\n",
    "        '''\n",
    "        预测方法\n",
    "        功能：使用集成模型对输入数据进行预测，返回均值和方差\n",
    "        实现细节：\n",
    "            分批处理大规模输入数据\n",
    "            将每个批次的数据复制到所有网络进行并行预测\n",
    "            合并所有网络的预测结果，返回整体均值和方差\n",
    "        输出含义：\n",
    "            均值：集成模型对下一状态的预测\n",
    "            方差：预测的不确定性估计，可用于衡量预测的可靠性\n",
    "        '''\n",
    "        # 对输入进行预测，集成多个模型的结果\n",
    "        mean, var = [], []\n",
    "        for i in range(0, inputs.shape[0], batch_size):\n",
    "            # 0, inputs.shape[0], batch_size 0 50 64\n",
    "\n",
    "            input = torch.from_numpy(inputs[i:min(i+batch_size, inputs.shape[0])]).float().to(device)\n",
    "            # input = torch.from_numpy(inputs[i:50]).float().to(device)\n",
    "            \n",
    "            # 对每个输入，使用所有模型进行预测\n",
    "            cur_mean, cur_var = self.model(input[None, :, :].repeat([self._num_network, 1, 1]), return_log_var = False)\n",
    "            mean.append(cur_mean.detach().cpu().numpy())\n",
    "            var.append(cur_var.detach().cpu().numpy())\n",
    "        return np.hstack(mean), np.hstack(var)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "859e0624",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FakeEnv:\n",
    "    def __init__(self, model):\n",
    "        # 用于预测下一状态和奖励的动力学模型\n",
    "        self.model = model\n",
    "\n",
    "    def step(self, obs, act):\n",
    "        '''\n",
    "        单步模拟方法\n",
    "        功能：模拟真实环境的单步交互，根据当前状态和动作预测下一状态和奖励\n",
    "        处理流程：\n",
    "            输入处理：将当前状态obs和动作act拼接作为模型输入\n",
    "            模型预测：调用集成模型预测下一状态和奖励的均值与方差\n",
    "            状态更新：对预测的状态增量（除奖励外）加上当前状态，得到完整的下一状态\n",
    "            采样：从预测的分布中采样，引入随机性以模拟环境的不确定性\n",
    "            模型选择：为每个样本随机选择一个集成中的模型结果\n",
    "            输出分离：将采样结果分离为奖励和下一状态并返回\n",
    "        '''\n",
    "        # 输入处理：将当前状态obs和动作act拼接作为模型输入\n",
    "        inputs = np.concatenate((obs, act), axis=-1)\n",
    "        # inputs.shape (50, 4)\n",
    "\n",
    "        # 模型预测：调用集成模型预测下一状态和奖励的均值与方差\n",
    "        ensemble_model_means, ensemble_model_vars = self.model.predict(inputs)\n",
    "        # ensemble_model_means.shape = (5, 50, 4) num_network：集成模型的数量（默认 5 个）\n",
    "        # ensemble_model_vars.shape = (5, 50, 4)\n",
    "\n",
    "        # 状态更新：对预测的状态增量（除奖励外）加上当前状态，得到完整的下一状态\n",
    "        ensemble_model_means[:, :, 1:] += obs.numpy()\n",
    "        '''\n",
    "        切片操作解析\n",
    "        1. 数据结构假设\n",
    "            ensemble_model_means 是一个三维数组，形状为 (num_models, batch_size, output_dim)。\n",
    "            num_models：集成模型中网络的数量。\n",
    "            batch_size：批量处理的样本数。\n",
    "            output_dim：模型输出的维度。\n",
    "        2. 切片含义\n",
    "            ensemble_model_means[:, :, 1:] 表示：\n",
    "            选择所有模型（第一维）。\n",
    "            选择所有批次样本（第二维）。\n",
    "            选择从索引 1 开始到最后的所有输出维度（第三维）。\n",
    "        3. 具体作用\n",
    "            在动力学模型预测中，通常：\n",
    "            ensemble_model_means[:, :, 0]：预测的奖励值（标量）。\n",
    "            ensemble_model_means[:, :, 1:]：预测的状态转移量（如位置、速度等）。\n",
    "        '''\n",
    "        ensemble_model_stds = np.sqrt(ensemble_model_vars)\n",
    "        ensemble_samples = ensemble_model_means + np.random.normal(size=ensemble_model_means.shape) * ensemble_model_stds\n",
    "        num_models, batch_size, _ = ensemble_model_means.shape\n",
    "        # 5, 50, 4 = ensemble_model_means.shape\n",
    "\n",
    "        models_to_use = np.random.choice([i for i in range(self.model._num_network)], size=batch_size)\n",
    "        # models_to_use [3 0 3 3 2 0 4 1 1 1 2 4 3 1 2 2 3 2 2 0 4 1 2 1 1 4 2 4 0 3 1 2 1 4 4 4 2 0 3 1 2 0 3 4 1 2 0 4 3 1]\n",
    "        batch_inds = np.arange(0, batch_size)\n",
    "        # batch_inds = [0 1 ... 49]\n",
    "\n",
    "        # 采样：从预测的分布中采样，引入随机性以模拟环境的不确定性\n",
    "        samples = ensemble_samples[models_to_use, batch_inds]\n",
    "        # samples.shape (50, 4)\n",
    "        '''\n",
    "        假设：\n",
    "        batch_size = 3\n",
    "        models_to_use = [2, 0, 1]（为 3 个样本分别选择模型 2、0、1）\n",
    "        batch_inds = [0, 1, 2]（即样本索引）\n",
    "        则 samples 会选择：\n",
    "        ensemble_samples[2, 0]（模型 2 对样本 0 的预测）\n",
    "        ensemble_samples[0, 1]（模型 0 对样本 1 的预测）\n",
    "        ensemble_samples[1, 2]（模型 1 对样本 2 的预测）\n",
    "        '''\n",
    "\n",
    "        # 输出分离：将采样结果分离为奖励和下一状态并返回\n",
    "        rewards, next_obs = samples[:, :1], samples[:, 1:]\n",
    "        return rewards, next_obs\n",
    "\n",
    "    def propagate(self, obs, actions):\n",
    "        '''\n",
    "        序列模拟方法\n",
    "        功能：模拟一个动作序列的执行，返回累积奖励\n",
    "        处理流程：\n",
    "            初始化：复制初始状态，初始化累积奖励\n",
    "            序列执行：按顺序执行动作序列中的每个动作\n",
    "            单步模拟：调用step方法预测每个动作的结果\n",
    "            累积奖励：累加每一步的奖励\n",
    "            状态更新：更新当前状态为预测的下一状态\n",
    "            返回结果：返回整个序列的累积奖励\n",
    "        设计特点与优势\n",
    "        不确定性建模：\n",
    "            利用集成模型的方差估计，通过随机采样模拟环境的随机性\n",
    "            随机选择不同模型的预测结果，增加模拟的多样性\n",
    "        高效批处理：\n",
    "            支持批量状态和动作的并行处理\n",
    "            适合在强化学习中进行大规模轨迹搜索\n",
    "        与真实环境接口一致：\n",
    "            step方法的输入输出与真实环境类似，便于无缝替换\n",
    "            可用于模型预测控制（MPC）等基于模型的强化学习算法\n",
    "        '''\n",
    "        with torch.no_grad():\n",
    "            obs = np.copy(obs)\n",
    "            total_reward = np.expand_dims(np.zeros(obs.shape[0]), axis=-1)\n",
    "            obs, actions = torch.as_tensor(obs), torch.as_tensor(actions)\n",
    "            # obs.shape, actions.shape: torch.Size([50, 3]) torch.Size([50, 25])\n",
    "            # obs.shape的维度(n_sequence：CEM 算法生成的候选动作序列数量，obs_dim：状态维度)\n",
    "            # actions.shape的维度(n_sequence：CEM 算法生成的候选动作序列数量, plan_horizon：规划 horizon，决定预测未来多少步)\n",
    "            for i in range(actions.shape[1]):\n",
    "                action = torch.unsqueeze(actions[:, i], 1)\n",
    "                rewards, next_obs = self.step(obs, action)\n",
    "                total_reward += rewards\n",
    "                obs = torch.as_tensor(next_obs)\n",
    "            return total_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "b5182ec1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CEM:\n",
    "    def __init__(self, n_sequence, elite_ratio, fake_env, upper_bound, lower_bound):\n",
    "        '''\n",
    "        类的初始化\n",
    "        核心参数：\n",
    "            n_sequence：每次迭代生成的动作序列数量\n",
    "            elite_ratio：精英样本比例，用于更新分布参数\n",
    "            upper_bound和lower_bound：动作空间的上下界\n",
    "            fake_env：基于模型的虚拟环境，用于评估动作序列\n",
    "        '''\n",
    "        # 每次迭代生成的动作序列数量\n",
    "        self.n_sequence = n_sequence\n",
    "         # 精英序列的比例\n",
    "        self.elite_ratio = elite_ratio\n",
    "        # 动作的上界\n",
    "        self.upper_bound = upper_bound\n",
    "        # 动作的下界\n",
    "        self.lower_bound = lower_bound\n",
    "        # 环境模型，用于预测状态转移和奖励\n",
    "        self.fake_env = fake_env\n",
    "\n",
    "    def optimize(self, state, init_mean, init_var):\n",
    "        ''' \n",
    "        优化方法\n",
    "        算法流程：\n",
    "            初始化：设置初始均值和方差，创建截断正态分布\n",
    "            迭代优化（5 轮）：\n",
    "            约束处理：根据动作边界调整方差，确保采样在有效范围内\n",
    "            采样：从截断正态分布生成动作序列\n",
    "            评估：在虚拟环境中执行动作序列，计算累积奖励\n",
    "            选择精英：根据奖励排序，选择表现最优的部分序列\n",
    "            更新分布：根据精英样本的统计特性更新分布参数\n",
    "            返回结果：最终分布的均值作为最优动作序列\n",
    "        '''\n",
    "        # 从初始均值和方差开始优化\n",
    "        mean, var = init_mean, init_var\n",
    "        # 创建截断正态分布采样器（限制在[-2,2]区间）\n",
    "        X = truncnorm(-2, 2, loc=np.zeros_like(mean), scale=np.ones_like(var))\n",
    "        # 复制状态用于批量评估多个动作序列\n",
    "        state = np.tile(state, (self.n_sequence, 1))\n",
    "\n",
    "        # 迭代优化5次\n",
    "        for _ in range(5):\n",
    "            # 计算当前均值到边界的距离，确保采样不会超出动作边界\n",
    "            lb_dist, ub_dist = mean - self.lower_bound, self.upper_bound - mean\n",
    "            # 基于边界距离约束方差，避免采样超出动作空间\n",
    "            constrained_var = np.minimum(np.minimum(np.square(lb_dist / 2), np.square(ub_dist / 2)), var)\n",
    "            # 从约束分布中采样生成动作序列\n",
    "            action_sequences = [X.rvs() for _ in range(self.n_sequence)] * np.sqrt(constrained_var) + mean\n",
    "            # X.rvs().shape (25,) 因为mean.shape, var.shape (25,) (25,)所以采样X.rvs().shape为(25,)\n",
    "            # action_sequences.shape (50, 25)\n",
    "\n",
    "            # 使用环境模型预测每个动作序列的累积奖励\n",
    "            returns = self.fake_env.propagate(state, action_sequences)[:, 0]\n",
    "            # 选择累积奖励最高的精英动作序列\n",
    "            elites = action_sequences[np.argsort(returns)][-int(self.elite_ratio * self.n_sequence):]\n",
    "            # 计算精英序列的统计量更新分布参数\n",
    "            new_mean = np.mean(elites, axis=0)\n",
    "            new_var = np.var(elites, axis=0)\n",
    "             # 平滑更新均值和方差，防止过快收敛\n",
    "            mean = 0.1 * mean + 0.9 * new_mean\n",
    "            var = 0.1 * var + 0.9 * new_var\n",
    "        return mean  # 返回优化后的动作序列均值作为最优动作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "254bbc0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    def __init__(self, capacity):\n",
    "        self.buffer = collections.deque(maxlen=capacity)\n",
    "    \n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        self.buffer.append((state, action, reward, next_state, done))\n",
    "\n",
    "    def size(self):\n",
    "        return len(self.buffer)\n",
    "    \n",
    "    def return_all_samples(self):\n",
    "        all_transitions = list(self.buffer)\n",
    "        state, action, reward, next_state, done = zip(*all_transitions)\n",
    "        return np.array(state), action, reward, np.array(next_state), done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "a7326512",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PETS:\n",
    "    '''PETS算法'''\n",
    "    def __init__(self, env, replay_buffer, n_sequence, elite_ratio, plan_horizon, num_episodes):\n",
    "        '''\n",
    "        类的定义和初始化\n",
    "        核心组件：\n",
    "            环境与数据：真实环境env和经验回放缓冲区replay_buffer\n",
    "            模型：集成动力学模型EnsembleDynamicsModel和基于模型的虚拟环境FakeEnv\n",
    "            规划器：交叉熵方法优化器CEM，用于在虚拟环境中寻找最优动作序列\n",
    "        超参数：\n",
    "            plan_horizon：规划 horizon，决定预测未来多少步\n",
    "            n_sequence：CEM 算法生成的候选动作序列数量\n",
    "            elite_ratio：CEM 算法中选择的精英序列比例\n",
    "        '''\n",
    "        self._env = env\n",
    "        self._env_pool = replay_buffer\n",
    "\n",
    "        obs_dim = env.observation_space.shape[0]\n",
    "        self._action_dim = env.action_space.shape[0]\n",
    "        # obs_dim, self._action_dim = 3 1\n",
    "        self._model = EnsembleDynamicsModel(obs_dim, self._action_dim)\n",
    "        self._fake_env = FakeEnv(self._model)\n",
    "        self.upper_bound = env.action_space.high[0]\n",
    "        self.lower_bound = env.action_space.low[0]\n",
    "        # self.upper_bound, self.lower_bound = 2.0 -2.0\n",
    "\n",
    "        self._cem = CEM(n_sequence, elite_ratio, self._fake_env, self.upper_bound, self.lower_bound)\n",
    "        self.plan_horizon = plan_horizon\n",
    "        self.num_episodes = num_episodes\n",
    "    \n",
    "    def train_model(self):\n",
    "        '''\n",
    "        模型训练方法\n",
    "        功能：从回放缓冲区获取数据，训练集成动力学模型\n",
    "        数据处理：\n",
    "            输入：当前状态与动作的拼接\n",
    "            标签：奖励与状态变化量的拼接\n",
    "            训练逻辑：调用EnsembleDynamicsModel的train方法，使用验证集进行早停和模型选择\n",
    "        '''\n",
    "        env_samples = self._env_pool.return_all_samples()\n",
    "        obs = env_samples[0]\n",
    "        actions = np.array(env_samples[1])\n",
    "        rewards = np.array(env_samples[2]).reshape(-1, 1)\n",
    "        next_obs = env_samples[3]\n",
    "        inputs = np.concatenate((obs, actions), axis=-1)\n",
    "        labels = np.concatenate((rewards, next_obs - obs), axis=-1)\n",
    "        self._model.train(inputs, labels)\n",
    "    \n",
    "    def mpc(self):\n",
    "        '''\n",
    "        基于模型预测控制（MPC）的决策方法\n",
    "        核心逻辑：\n",
    "        初始化：设置动作分布的初始均值和方差\n",
    "        循环交互：\n",
    "            使用 CEM 算法在虚拟环境中优化未来plan_horizon步的动作序列\n",
    "            执行最优动作序列中的第一个动作\n",
    "            收集真实环境反馈并存入回放缓冲区\n",
    "        滚动更新：将剩余动作序列前移，并为新的最后一步添加初始值\n",
    "        优势：通过模型预测多步未来，选择最优动作序列，具有前瞻性\n",
    "        '''\n",
    "        mean = np.tile((self.upper_bound + self.lower_bound) / 2.0, self.plan_horizon)\n",
    "        var = np.tile(np.square(self.upper_bound - self.lower_bound) / 16, self.plan_horizon)\n",
    "        # mean.shape, var.shape (25,) (25,)\n",
    "        obs, _ = self._env.reset()\n",
    "        done, episode_return = False, 0\n",
    "        while not done:\n",
    "            actions = self._cem.optimize(obs, mean, var)\n",
    "            # 选取第一个动作\n",
    "            action = actions[:self._action_dim]\n",
    "            next_obs, reward, terminated, truncated, info = self._env.step(action)\n",
    "            done = terminated or truncated\n",
    "            self._env_pool.add(obs, action, reward, next_obs, done)\n",
    "            obs = next_obs\n",
    "            episode_return += reward\n",
    "            mean = np.concatenate([np.copy(actions)[self._action_dim:], np.zeros(self._action_dim)])\n",
    "        return episode_return\n",
    "    \n",
    "    def explore(self):\n",
    "        ''' \n",
    "        随机探索方法\n",
    "        功能：执行随机策略收集初始数据\n",
    "        用途：在训练初期填充回放缓冲区，为动力学模型提供初始训练数据\n",
    "        '''\n",
    "        obs, _ = self._env.reset()\n",
    "        done, episode_return = False, 0\n",
    "        while not done:\n",
    "            action = self._env.action_space.sample()\n",
    "            next_obs, reward, terminated, truncated, info = self._env.step(action)\n",
    "            done = terminated or truncated\n",
    "            self._env_pool.add(obs, action, reward, next_obs, done)\n",
    "            obs = next_obs\n",
    "            episode_return += reward\n",
    "        return episode_return\n",
    "    \n",
    "    def train(self):\n",
    "        '''\n",
    "        整体训练流程\n",
    "        训练循环：\n",
    "        初始化：通过随机探索收集初始数据\n",
    "        迭代优化：\n",
    "            训练动力学模型\n",
    "            使用训练好的模型通过 MPC 进行决策，与真实环境交互\n",
    "            收集新数据存入回放缓冲区\n",
    "        持续改进：随着数据积累，模型和策略不断优化\n",
    "        '''\n",
    "        return_list = []\n",
    "        # 先进行随机策略的探索来收集一条序列的数据\n",
    "        explore_return = self.explore()\n",
    "        print('episode: 1, return: %d' % explore_return)\n",
    "        return_list.append(explore_return)\n",
    "\n",
    "        for i_episode in range(self.num_episodes - 1):\n",
    "            self.train_model()\n",
    "            episode_return = self.mpc()\n",
    "            return_list.append(episode_return)\n",
    "            print('episode: %d, return: %d' % (i_episode + 2, episode_return))\n",
    "        return return_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "929ebf85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 1, return: -1483\n",
      "episode: 2, return: -1028\n",
      "episode: 3, return: -1077\n",
      "episode: 4, return: -945\n",
      "episode: 5, return: -887\n",
      "episode: 6, return: -632\n",
      "episode: 7, return: -122\n",
      "episode: 8, return: -357\n",
      "episode: 9, return: -127\n",
      "episode: 10, return: -267\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAHHCAYAAABwaWYjAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAYexJREFUeJzt3XlYVOX7BvB7ZoBh32RVkEXcUNxAEfcFJcMSS0tLA5dKv1q5ZGqLZmVWZllZLlmipalptogbuZskiisqLomCyirLIDsz5/cHMr8mcRkcOLPcn+uaqzjznnOeIyq373nnORJBEAQQERER0UOTil0AERERkaFhgCIiIiLSEgMUERERkZYYoIiIiIi0xABFREREpCUGKCIiIiItMUARERERaYkBioiIiEhLDFBEREREWmKAIiISma+vL2JiYuq0b58+fdCnTx+d1kNED8YARWRkYmNjIZFI1C9LS0u0aNECkydPRlZWlnrcvn37NMb997V+/Xq8++679x1T8/r3D/A//vgDvXv3hpubG6ytreHv749nnnkGO3bsEOFXo3b/rl0qlaJx48YYOHAg9u3bJ3ZpJmXp0qUYPnw4mjZtColEUucQSSQGM7ELIKL68d5778HPzw9lZWU4dOgQli5dim3btiE5ORnW1tbqca+++io6d+581/5hYWEIDAxEQECAetvt27cxceJEDB06FE899ZR6u7u7OwDg008/xYwZM9C7d2/Mnj0b1tbWuHz5Mv7880+sX78ejz32WD1esXYGDBiAF154AYIgIDU1Fd988w369euHuLg4DBo0SOzyTMLHH3+MoqIidOnSBRkZGWKXQ6QVBigiIzVo0CCEhIQAAMaPH49GjRrhs88+w2+//YaRI0eqx/Xs2RPDhg2753HatWun/v/c3FxMnDgR7dq1w6hRozTGVVVV4f3338eAAQOwa9euu46TnZ39qJekUy1atNC4hqFDh6Jdu3ZYvHgxA1QD2b9/v3r2ydbWVuxyiLTCW3hEJqJfv34AgNTU1Ho5fm5uLhQKBbp3717r+25ubg88Rk0Ia9asGeRyOXx9ffHmm2+ivLxcY5yvry8GDx6MQ4cOoUuXLrC0tIS/vz/WrFlT5/qDgoLg4uKi8euTkpKCYcOGwdnZGZaWlggJCcHvv/+usV/NLdO//voL06ZNg6urK2xsbDB06FDk5ORojBUEAR988AG8vLxgbW2Nvn374uzZs3fVUnPr9L9qznX16tV7Xse9xtTcsv33bco+ffqgbdu2OH36NHr37g1ra2sEBARg06ZNAKoDTmhoKKysrNCyZUv8+eef9zwvAFRWVsLZ2Rljxoy56z2FQgFLS0u8/vrr6m0+Pj61XieRIWCAIjIR//zzDwCgUaNGGtuLioqQm5t710sQBK2O7+bmBisrK/zxxx/Iy8urU43jx4/HnDlz0KlTJ3z++efo3bs3FixYgBEjRtw19vLlyxg2bBgGDBiARYsWwcnJCTExMbUGkoeRn5+P/Px89a/P2bNn0bVrV5w/fx6zZs3CokWLYGNjg6ioKGzZsuWu/V955RWcOnUKc+fOxcSJE/HHH39g8uTJGmPmzJmDd955B+3bt8fChQvh7++PgQMHori4uE4160J+fj4GDx6M0NBQfPLJJ5DL5RgxYgQ2bNiAESNG4PHHH8dHH32E4uJiDBs2DEVFRfc8lrm5OYYOHYpff/0VFRUVGu/9+uuvKC8vr/V7SWSQBCIyKqtWrRIACH/++aeQk5MjpKenC+vXrxcaNWokWFlZCdevXxcEQRD27t0rALjnKyMj465j5+TkCACEuXPn1nruOXPmCAAEGxsbYdCgQcL8+fOFpKSkh6r75MmTAgBh/PjxGttff/11AYCwZ88e9TYfHx8BgHDgwAH1tuzsbEEulwvTp09/4LkACOPGjRNycnKE7Oxs4ciRI0L//v0FAMKiRYsEQRCE/v37C0FBQUJZWZl6P5VKJXTr1k1o3ry5elvNr3d4eLigUqnU26dOnSrIZDKhoKBAXZ+FhYUQGRmpMe7NN98UAAjR0dHqbXPnzhVq++u55lypqanqbb179xZ69+593zGC8P/f771792rsC0BYt26deltKSooAQJBKpcLff/+t3r5z504BgLBq1araf1H/M+6PP/7Q2P74448L/v7+99zPxsZG49eASN9xBorISIWHh8PV1RXe3t4YMWIEbG1tsWXLFjRp0kRj3Jw5cxAfH3/Xy9nZWetzzps3D+vWrUPHjh2xc+dOvPXWWwgODkanTp1w/vz5++67bds2AMC0adM0tk+fPh0AEBcXp7E9MDAQPXv2VH/t6uqKli1b4sqVKw9V63fffQdXV1e4ubkhNDRUfQtuypQpyMvLw549e/DMM89ozNDdunULERERuHTpEm7cuKFxvJdeeknjdlTPnj2hVCpx7do1AMCff/6JiooKvPLKKxrjpkyZ8lD11hdbW1uNWaGWLVvC0dERrVu3RmhoqHp7zf8/6Ne3X79+cHFxwYYNG9Tb8vPzER8fj2effVbH1ROJh4vIiYzU119/jRYtWsDMzAzu7u5o2bIlpNK7/80UFBSE8PBwnZ135MiRGDlyJBQKBY4cOYLY2FisW7cOTzzxBJKTk2FpaVnrfteuXYNUKtX41B8AeHh4wNHRUR1EajRt2vSuYzg5OSE/P/+h6hwyZAgmT54MiUQCOzs7tGnTBjY2NgCqbw8KgoB33nkH77zzTq37Z2dna4TR/9bj5OQEAOp6aupv3ry5xjhXV1f1WDF4eXndtQ7JwcEB3t7ed20D/v96Kioq7rpV6+rqCjMzMzz99NNYt24dysvLIZfL8csvv6CyspIBiowKAxSRkerSpYv6U3hisLe3x4ABAzBgwACYm5tj9erVOHLkCHr37n3f/R52UbFMJqt1u/CQa7e8vLzuGRxVKhUA4PXXX0dEREStY/4b9B61nn+716+BUqnU+b73qvtB13P48GH07dtX473U1FT4+vpixIgRWL58ObZv346oqChs3LgRrVq1Qvv27R9YP5GhYIAionoXEhKC1atX37fXj4+PD1QqFS5duoTWrVurt2dlZaGgoAA+Pj4NUSoAwN/fH0D1omhdzc7V1H/p0iX18QEgJyfnrlmzmhmpgoICODo6qrf/dxauNv/e998eZl9ttG/fHvHx8RrbPDw8AAC9evWCp6cnNmzYgB49emDPnj146623dHp+IrFxDRQR6URJSQkSEhJqfW/79u0AqtfX3Mvjjz8OAFi8eLHG9s8++wwAEBkZqYMqH46bmxv69OmD5cuX1xr6/tue4GGEh4fD3NwcX331lcas1H+vFwCaNWsGADhw4IB6W3FxMVavXv3A89S2r1KpxIoVK7Su+X6cnJwQHh6u8aq5PSuVSjFs2DD88ccf+OGHH1BVVcXbd2R0OANFZOIOHjyIsrKyu7a3a9dOo4nmg5SUlKBbt27o2rUrHnvsMXh7e6OgoAC//vorDh48iKioKHTs2PGe+7dv3x7R0dFYsWIFCgoK0Lt3byQmJmL16tWIioq663ZRffv666/Ro0cPBAUF4cUXX4S/vz+ysrKQkJCA69ev49SpU1odz9XVFa+//joWLFiAwYMH4/HHH8eJEyewfft2uLi4aIwdOHAgmjZtinHjxmHGjBmQyWT4/vvv4erqirS0tPuep02bNujatStmz56NvLw8ODs7Y/369aiqqtL61+BRPPvss/jqq68wd+5cBAUFacwq1vjjjz/Uv46VlZU4ffo0PvjgAwDAk08+qdXvP6KGxgBFZOK+/PLLWrfPnTtXqx9gjo6O+PbbbxEXF4dVq1YhMzMTMpkMLVu2xMKFC/Hqq68+8BgrV66Ev78/YmNjsWXLFnh4eGD27NmYO3fuQ9ehK4GBgTh27BjmzZuH2NhY3Lp1C25ubujYsSPmzJlTp2N+8MEHsLS0xLJly7B3716EhoZi165dd82umZubY8uWLfjf//6Hd955Bx4eHpgyZQqcnJxqbVL5X2vXrsXLL7+Mjz76CI6Ojhg3bhz69u2LAQMG1KnuuujWrRu8vb2Rnp5+z9mnzZs3a8yqnThxAidOnABQvUaNAYr0mUSoywpHIiIiIhPGNVBEREREWmKAIiIiItISAxQRERGRlhigiIiIiLTEAEVERESkJQYoIiIiIi2xD1Q9UKlUuHnzJuzs7B76uV5EREQkLkEQUFRUhMaNG9f68PV/Y4CqBzdv3rzrSeZERERkGNLT0+Hl5XXfMQxQ9cDOzg5A9TfA3t5e5GqIiIjoYSgUCnh7e6t/jt8PA1Q9qLltZ29vzwBFRERkYB5m+Q0XkRMRERFpiQGKiIiISEsMUERERERaYoAiIiIi0hIDFBEREZGWGKCIiIiItMQARURERKQlBigiIiIiLTFAEREREWmJAYqIiIhISwxQRERERFpigCIiIiLSEgMUERHVC5VKQGmFUuwyiOoFAxQREdWLcauPosv8P5F0LU/sUoh0jgGKiIh07nyGAnsv5KCovAovrknC1dxisUsyeT8lpuHxLw7i1xM3IAiC2OUYPAYoIiLSufWJaer/zyuuwJjYo8gvrhCxItO29fRNzP7lDM5lKDBlw0lMXneC349HxABFREQ6VVqhxC8nbgAAPnumPZo4WiE1txgv/5CE8iquiWpoR67cwrQNpwAAXXydYSaVIO5MBgYuPoA9KVkiV2e4GKCIiEin4s5koKisCt7OVojq0ASrxnSGndwMiVfzMOPn01CpePuooVzKKsKLa46hQqlCRBt3/PRSV2z5X3cEuNkip6gcY2OPYfYvZ1BcXiV2qQaHAYqIiHTqpzu370Z0bgqpVIIW7nZYNjoYZlIJfj91E5/FXxS5QtOQpShDzKqjUJRVIdjHCV+M6AiZVIIgLwdsfaUHxvfwg0RS/f0a9MVBHL3Kxf7aYIAiIiKduZBZhKRr+TCTSjA8xEu9vXuACz58KggAsGTvZWw8mi5WiSahqKwSMauO4kZBKfxdbLDyhRBYmsvU71uay/D24ECsG98VTRytkJZXgmeWJ2DB9vO8zfqQGKCIiEhnamafwlu7w83OUuO9Z0K88Uq/AADAm1vO4NCl3AavzxRUKlX439rjOJ+hgIutBVaP7QInG4tax4Y1a4TtU3piWLAXBAFYvv8Khiz5C+duKhq4asPDAEVERDpRVqnEL8evAwBGhjatdcy0AS0wpENjVKkETPwxCRcyixqyRKMnCAJmbT6Dg5dyYW0hw/cxneHtbH3ffewtzfHp8PZYPjoYjWwskJJZhCFfH8I3+y5DyfVq98QARUREOrHtTAYUZVVo4miFngEutY6RSCT4ZFg7dPF1RlF5FcbGHkW2oqyBKzVen8dfxObj1yGTSvD1c53QzsvxofeNaOOBnVN7YUCgOyqVAj7ZcQHPLE9gD697YIAiIiKdqLl9N7KLN6RSyT3Hyc1kWPFCMPxdbHCjoBTjVh9DSQU/BfaofkpMw5d7LgMA5ke1Rd9Wblofw8VWjhWjg/Hp8PawlZsh6Vo+Hv/yINYeucbmm//BAEVERI/sUlYRjl7Nh0wqwfAQ7weOd7S2wKoxneFsY4EzNwrx6k8neLvoEexJycLbvyYDAF7t3xwjutR+C/VhSCQSDAv2wo4pPRHm3wglFUq8tSUZMauOIouzhWoMUERE9Mh+Sqz+VF3/Vm5wt7d8wOhqPo1s8O0LIbAwk+LP89l4f+u5+izRaJ1KL8CktdUBdFiwF6aGN9fJcb2crLF2fCjeGRwICzMp9l/MwcDPD+CPUzd1cnxDxwBFRESPpKxSic0PWDx+L8E+Tvj8mQ4AgNjDV/H9oVRdl2fUrt0qxtjYoyitVKJncxcseCoIEsm9b59qSyqVYFwPP8S90gNBTRxQWFqJV346gVd/OoGCEtN+FAwDFBERPZIdyZkoLK1EE0cr9GruqvX+ke08MXtQKwDA+3HnsOtspq5LNEp5xRWIWXUUt4or0KaxPZaOCoa5rH5+rDd3t8Mv/+uG1/o3h+xOQ9SIxQew/2JOvZzPEDBAERHRI1l3Z/H4s529IbvP4vH7eamXP54LbQpBAF5dfwKn0gt0WKHxKa1QYtzqo0jNLUYTRyusiukMW7lZvZ7TXCbF1AEt8MvEbvB3tUGWohzR3yfi7V/PmOSHAIwmQF29ehXjxo2Dn58frKys0KxZM8ydOxcVFZpTjKdPn0bPnj1haWkJb29vfPLJJ3cd6+eff0arVq1gaWmJoKAgbNu2raEug4jIoFzOvo3E1DxIJdWNMutKIpHgvSfboHcLV5RVqjBu9TGk55XosFLjoVQJeG39CZxIK4CDlTlWj+0Mt4dcd6YL7b0dEfdKT8R08wUA/Ph3Gh7/4iCSruU3WA36wGgCVEpKClQqFZYvX46zZ8/i888/x7Jly/Dmm2+qxygUCgwcOBA+Pj5ISkrCwoUL8e6772LFihXqMYcPH8bIkSMxbtw4nDhxAlFRUYiKikJycrIYl0VEpNfW35l96tfKHR4Oj/ZD3EwmxdfPd0JrT3vk3i7H2NijKCyt1EWZRkMQBMz74yx2ncuChZkUK6NDEOBm1+B1WFnI8O6TbfDjuFB4Olji6q0SDF92GAt3pqCiStXg9YhBIhhxY4eFCxdi6dKluHLlCgBg6dKleOutt5CZmQkLi+q29rNmzcKvv/6KlJQUAMCzzz6L4uJibN26VX2crl27okOHDli2bNlDnVehUMDBwQGFhYWwt7fX8VUREemHskolwhbsRn5JJb6PCUG/Vu46OW5GYSmivv4LWYpydGvWCLFjusDCzGj+vf9Ilu3/Bx9tT4FEAnz9XCc8HuQpdkkoLK3EvN/P4pcTNwAAgZ72+PzZDmjp0fDB7lFp8/PbqH9HFhYWwtnZWf11QkICevXqpQ5PABAREYELFy4gPz9fPSY8PFzjOBEREUhISLjnecrLy6FQKDReRETGbufZTOSXVMLTwRK9W2jftPFePB2s8H1MZ9hYyHD4n1t4c8sZNnEE8NvJG/hoe/U/9t+ODNSL8AQADlbm+OzZDlj6fCc4WZvjXIYCT3x1CCsO/GPUvb2MNkBdvnwZX331FV5++WX1tszMTLi7a/4LqebrzMzM+46peb82CxYsgIODg/rl7V33dQBERIbiJx0sHr+XNo0dsOT5TpBJJdiUdB1L7nTYNlWH/8nF6z+fAgCM6+GHcT38RK7oboOCPLFzai/0b+WGCqUKH25LwcgVfxvtWja9D1CzZs2CRCK576vm9luNGzdu4LHHHsPw4cPx4osv1nuNs2fPRmFhofqVnp5e7+ckIhLTlZzb+PvKoy8ev5++Ld0w78k2AIBF8Rfx651bRKYmJVOBl9ckoVIpIDLIE2893lrsku7Jzc4SK6ND8PHTQbCxkCHxah4eW3wA6xPTjG4WsX4/86gD06dPR0xMzH3H+Pv7q///5s2b6Nu3L7p166axOBwAPDw8kJWVpbGt5msPD4/7jql5vzZyuRxyufyB10JEZCzWH63+h2Lflm5o7GhVb+cZ1dUH6XklWH7gCt7YdBqeDpYI9W9Ub+fTNxmFpRiz6iiKyqvQxdcZi55pf9/nDOoDiUSCZzs3RbdmLpi+8RQSr+Zh1i9nEH8uCwueDoKbXcN9YrA+6f0MlKurK1q1anXfV82aphs3bqBPnz4IDg7GqlWrIJVqXl5YWBgOHDiAysr//1RHfHw8WrZsCScnJ/WY3bt3a+wXHx+PsLCwer5SIiLDUF6lxKakO53HH+GZaw9r5mOt8HiQByqUKrz0QxL+ybld7+fUB4qySoxZdRQZhWUIcLPFiheCYWkuE7ush+btbI2fXuqKtx5vDQuZFLtTshHx+QFsP5Mhdmk6ofcB6mHVhKemTZvi008/RU5ODjIzMzXWLj333HOwsLDAuHHjcPbsWWzYsAFffPEFpk2bph7z2muvYceOHVi0aBFSUlLw7rvv4tixY5g8ebIYl0VEpHd2nc1CXnEFPOwt0ael9p3HtSWVSvDZMx3QsakjCkurQ8Wt2+X1fl4xVVSpMOGHJKRkFsHVTo7YMZ3haG3x4B31jEwqwYu9/PHHKz0Q6GmP/JJKTFx7HFM3nDT4FhVGE6Di4+Nx+fJl7N69G15eXvD09FS/ajg4OGDXrl1ITU1FcHAwpk+fjjlz5uCll15Sj+nWrRvWrVuHFStWoH379ti0aRN+/fVXtG3bVozLIiLSOzWLx5/p7A2zenp0yH9Zmsvw7QshaOpsjbS8EoxfcwxllcoGOXdDEwQBb2w6hcP/3IKNhQyrYjrDy8la7LIeSUsPO/w6qTsm9w2AVAJsOXEDjy0+gEOXcsUurc6Mug+UWNgHioiMVWpuMfp+ug8SCXBoZj80qcf1T7X5J+c2nvrmMApLKzGorQe+fq6T3q8J0tbHO1KwdN8/MJNK8H1MZ/RqUf+zfA3peFo+pm88hdTcYgBATDdfzHysFawsxL89yT5QRERUL9YfrZ596tPCtcHDEwA0c7XF8tHBMJdJsD05Ex/vSHnwTgbkh4SrWLrvHwDAgqeCjC48AUCnpk6Ie7UHRnf1AQDEHr6KyK8O4qSBPf+QAYqIiB5KRZUKm4413OLxe+nq3wgLh7UHACw/cAU//n1NtFp0adfZTMz9/SwAYNqAFhheT+0h9IG1hRnej2qL1WO7wN1ejis5xXh66WF8Fn8RlUrDeBQMAxQRET2U+HNZuFVcATc7Ofq10l3n8bqI6tgE0wa0AADM+S0Zey9ki1rPozqelo9X15+ASgBGdPbGK/0CxC6pQfRu4YpdU3pjSIfGUKoEfLn7EoZ+8xcuZRWJXdoDMUAREdFD+Xfn8YZaPH4/r/QLwLBgL6gEYPLa4zh30zAfo5WaW4zxq4+hrFKFvi1d8UFUW0gkxrWu634crM3xxYiOWPJcRzhamyP5hgKRXx3CyoNXoNLjR8GI/yeAiIj03rVbxTh0OReSeuw8ri2JRIIPhwahW7NGKK5QYmzsUWQUlopdllZyb5cjZlUi8oor0M7LAUue66QX4VQMg9s1xs4pvdCnpSsqqlT4IO48nlv5N67n6+ejYEzzu0RERFqp6Tzeq7krvJ315yP1FmZSLB0VjOZutshUlGFs7DHcLq8Su6yHUlJRhXGxR3HtVgm8na3wXXRn2Mj1/gEh9crd3hKrYjrjw6FBsLaQ4e8reXhs8UH8fCxd7x4FwwBFRET3VVGlws/HqgOUmIvH78XByhzfx3SGi60c5zMUmLT2OKr0fCFylVKFV9adwKnrhXCyNsfqMV3gasdHggHVM4vPhTbF9td6IsTHCbfLqzBj02m89EMScvWogSoDFBER3dfu81nIvV0BVzs5+rcWd/H4vXg7W+O76BBYmkux/2IO5vx+Vu9mLGoIgoB3fjuL3SnZkJtJsTI6BP6utmKXpXd8Gtlgw8thmPlYK5jLJIg/l4WIzw9g59nMB+/cABigiIjovtbVdB4P8YK5Hq/Pae/tiC9HdIREAqw7koYVB66IXVKtvtn3D35KTINEAnwxoiOCfZzFLklvyaQSTOzTDL9P7oFWHna4VVyBl39IwvSNp6AoE/dRMPr7J4GIiESXnleCg3cetzGis/7dvvuvgW088E5kIABgwfYUxJ3WrwfXbk66joU7LwAA3n2iDR5r6yFyRYahtac9fpvcHRP7NINUAmw+fh3TNpwUtSYGKCIiuqeazuM9m7vo1eLx+xnbww8x3XwBAFM3nkTStXxxC7rj4KUczNx8GgDwci9/RN+pkR6O3EyGmY+1wsaXwxDgZovXI1qKWg8DFBER1apSqcLGO53Hn9PDxeP3887gQIS3dkNFlQovrjmGa7eKRa3n3E0FJv54HFUqAU+0b4yZj7UStR5DFuLrjF1TeqGVh7jPmmWAIiKiWu0+n42conK42MoRHugudjlakUkl+HJkRwQ1cUBecQXGrDqKgpIKUWq5UVCKMbGJuF1eha7+zvh0eDujewByQ9OHXz8GKCIiqlVN5/Hher54/F6sLczwXXQImjha4UpuMV5ak4TyKmWD1lBYUomY7xORpShHC3dbLB8dArmZrEFroPpheH8iiIio3qXnleDApRwA1c9mM1Ru9pb4PqYz7ORmSLyahzc2nW6w9gblVUq8+MMxXMq+DXd7OWLHdIGDlXmDnJvqHwMUERHdZeOxdAgC0CPABT6NbMQu55G09LDD0lHBMJNK8NvJm/gs/mK9n1OlEjB94ykkpubBVm6G2DFd0NjRqt7PSw2HAYqIiDRUKVXYcFR/O4/XRY/mLvhwaBAA4Ks9l7HxTmf1+vLRjhRsPZ0BM6kEy0cHo7WnuAueSfcYoIiISMOelGxkF5WjkY0FBhjY4vH7eaazNyb3DQAAvPnLGfx1ObdezrPqr1R1E89PhrVD9wCXejkPiYsBioiINNQsHh8W4gULM+P6MTF9YAs82b4xqlQCJvyQhItZRTo9/o7kDLy39RwAYEZESzzVyUunxyf9YVx/MoiI6JHcKCjFvos1i8eN4/bdv0kkEiwc3g5dfJ1RVF6FMauOIruoTCfHPnY1D6+tPwlBAJ4PbYr/9Wmmk+OSfmKAIiIitQ1HqxePd2vWCH4uhr14/F7kZjIsHx0MPxcb3CgoxfjVx1BSUfVIx/wn5zbGrzmG8ioVwlu7Yd6TbSCRiN+riOoPAxQREQGoXjy+0cgWj9+Lk40FVsV0hrONBU5fL8SrP52EUlW39gbZRWWI/j4RBSWV6ODtiK9GdoKZAfbNIu3wO0xERACAfRdykKkog7ONBQa2MZ7F4/fi62KDb18IhoWZFH+ez8IHcee0PkZxeRXGxh7F9fxS+DayxnfRIbCyYKNMU8AARUREAP61eDzYy2S6ZQf7OOPzZzoAAFb9dRWr/kp96H0rlSr8b+1xJN9QwNnGArFjuqCRrbyeKiV9wwBFRES4WVCKvReyARh25/G6iGzniVmDqh/u+97Wc4g/l/XAfQRBwNtbkrH/Yg4szaX4LjoEvka6ZoxqxwBFRETYeCwdKgHo6u8Mf1dbsctpcC/38sfILk0hCMCrP53A6esF9x3/5e7L2HAsHVIJ8NXITujY1KlhCiW9wQBFRGTilCrB6DqPa0sikeD9IW3Qq4UrSiuVGBt7DNfzS2odu/FYOj7/s/pxMO8NaWtUzUbp4TFAERGZuP0Xs5FRWAYna3NEtPEQuxzRmMmk+Pq5jmjlYYfc2+UYG3sUirJKjTH7LmRj9i9nAAD/69MMo7r6iFEq6QEGKCIiE7fuSPXs09OdvGBpbhqLx+/FztIcq8Z0hru9HBezbmPij0moqFIBAJJvFOJ/a49DqRIwtGMTzIhoKXK1JCYGKCIiE5ZZWIY9KdWLpkeY6O27//J0sMJ30Z1hbSHDX5dv4a0tZ5CeV4KYVUdRUqFE94BG+PjpdmyUaeIYoIiITFjN4vEufs4IcDO9xeP30raJA75+rhOkEuDnpOuI/PIgcm+Xo5WHHZaOCja6ZwSS9vg7gIjIRP178fhznH26S99Wbpg3pC0AQFFWBU8HS8SO6QJ7S3ORKyN9YCZ2AUREJI4Dl3Jwo6AUDlbmeKyt6S4ev5/RXX1QXF6F+HNZ+HBoEDwcLMUuifQEAxQRkYn66Uh153EuHr+/Cb2bYULvZmKXQXqGt/CIiExQlqIMu1OqO4+P7GJanceJdIEBiojIBP18LB1KlYDOvk5o7m4ndjlEBocBiojIxKhUAn5KNO3O40SPigGKiMjEHLycixsFpbC3NMPjQZ5il0NkkIwyQJWXl6NDhw6QSCQ4efKkxnunT59Gz549YWlpCW9vb3zyySd37f/zzz+jVatWsLS0RFBQELZt29ZAlRMR1b+axeNPcfE4UZ0ZZYB644030Lhx47u2KxQKDBw4ED4+PkhKSsLChQvx7rvvYsWKFeoxhw8fxsiRIzFu3DicOHECUVFRiIqKQnJyckNeAhFRvchWlOHP89Wdx3n7jqjujC5Abd++Hbt27cKnn35613tr165FRUUFvv/+e7Rp0wYjRozAq6++is8++0w95osvvsBjjz2GGTNmoHXr1nj//ffRqVMnLFmypCEvg4ioXvycdB1VKgHBPk5o6cHF40R1ZVQBKisrCy+++CJ++OEHWFtb3/V+QkICevXqBQsLC/W2iIgIXLhwAfn5+eox4eHhGvtFREQgISHhnuctLy+HQqHQeBER6RuVSsD6o9W37zj7RPRojCZACYKAmJgYTJgwASEhIbWOyczMhLu7u8a2mq8zMzPvO6bm/dosWLAADg4O6pe3N3uqEJH++eufXKTnlcLO0gyRXDxO9Ej0PkDNmjULEonkvq+UlBR89dVXKCoqwuzZsxu8xtmzZ6OwsFD9Sk9Pb/AaiIge5KfEO4vHOzaBlQUXjxM9Cr1/lMv06dMRExNz3zH+/v7Ys2cPEhISIJfLNd4LCQnB888/j9WrV8PDwwNZWVka79d87eHhof5vbWNq3q+NXC6/67xERPokp6gcu87eWTweytt3RI9K7wOUq6srXF1dHzjuyy+/xAcffKD++ubNm4iIiMCGDRsQGhoKAAgLC8Nbb72FyspKmJtXP007Pj4eLVu2hJOTk3rM7t27MWXKFPWx4uPjERYWpsOrIiJqWJvuLB7v2NQRrTzsxS6HyODpfYB6WE2bav6LytbWFgDQrFkzeHl5AQCee+45zJs3D+PGjcPMmTORnJyML774Ap9//rl6v9deew29e/fGokWLEBkZifXr1+PYsWMarQ6IiAwJF48T6Z7er4HSJQcHB+zatQupqakIDg7G9OnTMWfOHLz00kvqMd26dcO6deuwYsUKtG/fHps2bcKvv/6Ktm3bilg5EVHdJVy5hWu3SmAnN8Pgdlw8TqQLEkEQBLGLMDYKhQIODg4oLCyEvT2nyolIXJPWHUfc6QyM7uqD96P4j0Gie9Hm57dJzUAREZma3Nvl2HW2ug0Lb98R6Q4DFBGREducdB2VSgHtvR0R2Jgz4kS6wgBFRGSkBEFQ9356rgsb/BLpEgMUEZGRSrhyC1dvlcBWbobB7e5+wDoR1R0DFBGRkfopsfqpCEM6NIaN3Gi61hDpBQYoIiIjdOt2OXYmc/E4UX1hgCIiMkK/HL+BCqUK7bwc0LaJg9jlEBkdBigiIiPz78XjnH0iqh8MUERERuZIah6u5BbDxkKGJ9pz8ThRfWCAIiIyMjWzT092aAxbLh4nqhcMUERERiS/uALbz3DxOFF9Y4AiIjIim49fR4VShTaN7RHExeNE9YYBiojISPx38bhEIhG5IiLjxQBFRGQkjl7Nxz85xbAyl2FIBy4eJ6pPDFBEREZCvXi8fWPYWZqLXA2RcWOAIiIyAgUlFYg7kwEAGBnKxeNE9Y0BiojICPxy/AYqqlRo7WmP9l5cPE5U3xigiIgM3L8Xjz/XxZuLx4kaAAMUEZGBS7qWj0vZt2FpLsWQjk3ELofIJDBAEREZuHV3Zp+eaNcY9lw8TtQgGKCIiAxYYUkl4k5z8ThRQ2OAIiIyYFtOXEd5lQqtPOzQ0dtR7HKITAYDFBGRgapePJ4OgJ3HiRoaAxQRkYE6nlaAC1lFkJtJEcXF40QNigGKiMhA1bQuGNyuMRysuHicqCExQBERGaDC0kpsPX0TAPBcqLfI1RCZHgYoIiID9NvJGyirVKGFuy06NXUSuxwik8MARURkYARBwLoj1bfvuHicSBwMUEREBuZkegFSMqsXjw/l4nEiUTBAEREZmJrF45FBnnC0thC5GiLTxABFRGRAFGWV+OMUO48TiY0BiojIgPx28iZKK5UIcLNFiA8XjxOJhQGKiMhAcPE4kf5ggCIiMhCnrxfifIYCFmZSPMXF40SiYoAiIjIQNYvHH2/rAScbLh4nEhMDFBGRASgqq8Tvp6o7j4/swsXjRGJjgCIiMgC/n7qJkgol/F1t0MXPWexyiEweAxQRkQGouX33HBePE+kFowtQcXFxCA0NhZWVFZycnBAVFaXxflpaGiIjI2FtbQ03NzfMmDEDVVVVGmP27duHTp06QS6XIyAgALGxsQ13AURE/3HmeiGSbyhgIZPiqU5eYpdDRADMxC5AlzZv3owXX3wRH374Ifr164eqqiokJyer31cqlYiMjISHhwcOHz6MjIwMvPDCCzA3N8eHH34IAEhNTUVkZCQmTJiAtWvXYvfu3Rg/fjw8PT0REREh1qURkQlbd2f26bG2HnDm4nEivSARBEEQuwhdqKqqgq+vL+bNm4dx48bVOmb79u0YPHgwbt68CXd3dwDAsmXLMHPmTOTk5MDCwgIzZ85EXFycRvAaMWIECgoKsGPHjoeqRaFQwMHBAYWFhbC3t3/0iyMik3W7vAqh8/9EcYUSP73YFWHNGoldEpHR0ubnt9Hcwjt+/Dhu3LgBqVSKjh07wtPTE4MGDdIIQgkJCQgKClKHJwCIiIiAQqHA2bNn1WPCw8M1jh0REYGEhISGuRAion/549RNFFco4edig67+XDxOpC+MJkBduXIFAPDuu+/i7bffxtatW+Hk5IQ+ffogLy8PAJCZmakRngCov87MzLzvGIVCgdLS0lrPXV5eDoVCofEiItKFmsXjI7t4c/E4kR7R+wA1a9YsSCSS+75SUlKgUqkAAG+99RaefvppBAcHY9WqVZBIJPj555/rtcYFCxbAwcFB/fL29q7X8xGRaUi+UYjT1wthLpPgaS4eJ9Irer+IfPr06YiJibnvGH9/f2RkVD+dPDAwUL1dLpfD398faWnV/4Lz8PBAYmKixr5ZWVnq92r+W7Pt32Ps7e1hZWVV6/lnz56NadOmqb9WKBQMUUT0yGpmnyLaeKCRrVzkaojo3/Q+QLm6usLV1fWB44KDgyGXy3HhwgX06NEDAFBZWYmrV6/Cx8cHABAWFob58+cjOzsbbm5uAID4+HjY29urg1dYWBi2bdumcez4+HiEhYXd89xyuRxyOf9yIyLdKS6vwm8nqzuPP8fO40R6R+9v4T0se3t7TJgwAXPnzsWuXbtw4cIFTJw4EQAwfPhwAMDAgQMRGBiI0aNH49SpU9i5cyfefvttTJo0SR2AJkyYgCtXruCNN95ASkoKvvnmG2zcuBFTp04V7dqIyPRsPX0Tt8ur4NvIGl39+ck7In2j9zNQ2li4cCHMzMwwevRolJaWIjQ0FHv27IGTkxMAQCaTYevWrZg4cSLCwsJgY2OD6OhovPfee+pj+Pn5IS4uDlOnTsUXX3wBLy8vrFy5kj2giKhBrUtMBwCM6NIUUikXjxPpG6PpA6VP2AeKiB7F2ZuFiPzyEMxlEiTM7g8Xrn8iahAm2QeKiMhYrL8z+zQw0IPhiUhPMUAREemRC5lF2JR0HQAwkovHifQWAxQRkZ4oLKnESz8cQ2mlEj0CXNCNj20h0lsMUEREekCpEvDahhO4dqsEXk5W+GpkRy4eJ9JjDFBERHpg8Z8Xse9CDuRmUiwbFQwnGwuxSyKi+2CAIiIS2Y7kTHy15zIA4KOng9C2iYPIFRHRgzBAERGJ6HJ2EaZvPAkAGNvdD0M78pl3RIaAAYqISCSKskq8tCYJxRVKdPV3xuzHW4ldEhE9JAYoIiIRqFQCpm04iSu5xWjsYIklz3WCuYx/JRMZCv5pJSISwZd7LuHP89mwMJNi2ehgNswkMjAMUEREDWz3+Sws/vMSAGB+VFu083IUtyAi0hoDFBFRA7qScxtT1p8EALwQ5oPhId7iFkREdcIARUTUQG6XV+GlH5JQVF6Fzr5OeDsyUOySiKiOGKCIiBqASiVg+saTuJx9G+72cnz9fCdYmPGvYCJDxT+9REQNYOn+f7DzbBYsZFIsHRUMNztLsUsiokfAAEVEVM/2XsjGp7suAADeG9IGnZo6iVwRET0qBigionp0NbcYr/10AoIAPBfaFCO6NBW7JCLSAQYoIqJ6UlxehZd/SIKirAqdmjpi7hNcNE5kLBigiIjqgSAIeGPzaVzIKoKrnRxLRwVDbiYTuywi0hEGKCKierDiwBXEnc6AmVSCpc93grs9F40TGRMGKCIiHTt4KQcf70gBAMx9sg1CfJ1FroiIdK1OAaq0tBQlJSXqr69du4bFixdj165dOiuMiMgQpeeV4JWfTkAlAM+EeGFUKBeNExmjOgWoIUOGYM2aNQCAgoIChIaGYtGiRRgyZAiWLl2q0wKJiAxFaYUSL/2QhIKSSrT3csB7Q9pCIpGIXRYR1YM6Bajjx4+jZ8+eAIBNmzbB3d0d165dw5o1a/Dll1/qtEAiIkMgCAJm/XIa5zMUcLG1wNJRwbA056JxImNVpwBVUlICOzs7AMCuXbvw1FNPQSqVomvXrrh27ZpOCyQiMgTfHUrFbydvwkwqwdfPdUJjRyuxSyKielSnABUQEIBff/0V6enp2LlzJwYOHAgAyM7Ohr29vU4LJCLSd4f/ycWC7dWLxt+ObI1Q/0YiV0RE9a1OAWrOnDl4/fXX4evri9DQUISFhQGono3q2LGjTgskItJnNwpKMXndCShVAp7q2ATR3XzFLomIGoBEEAShLjtmZmYiIyMD7du3h1RancMSExNhb2+PVq1a6bRIQ6NQKODg4IDCwkLOyBEZsbJKJYYvS8CZG4Vo09gemyd247onIgOmzc9vs7qexMPDAx4eHhrbunTpUtfDEREZFEEQ8OaWMzhzoxBO1uZYPpqLxolMSZ0CVHFxMT766CPs3r0b2dnZUKlUGu9fuXJFJ8UREemrNQnX8MvxG5BKgK+f6wQvJ2uxSyKiBlSnADV+/Hjs378fo0ePhqenJ/ucEJFJOXLlFt7feg4A8ObjrdEtwEXkioioodUpQG3fvh1xcXHo3r27rushItJrGYWlmLTuOKpUAp5s3xjjeviJXRIRiaBOn8JzcnKCszOf7UREpqW8SokJPx5H7u0KtPa0x8dPt+MMPJGJqlOAev/99zFnzhyN5+ERERkzQRAw59ezOJVeAAcrcywfFQwrCy4aJzJVdbqFt2jRIvzzzz9wd3eHr68vzM3NNd4/fvy4ToojItIX6xLTsOFYOqQS4KuRHdG0EReNE5myOgWoqKgoHZdBRKS/kq7l4d3fzwIAZkS0Qq8WriJXRERi0zpAVVVVQSKRYOzYsfDy8qqPmoiI9EaWogwTfjyOSqWAx4M8MKG3v9glEZEe0HoNlJmZGRYuXIiqqqr6qIeISG9UVKkw8cck5BSVo4W7LRYOa89F40QEoI6LyPv164f9+/fruhYiIr0y74+zOJ5WAHtLM6wYHQIbeZ0f3kBERqZOAWrQoEGYNWsWXn/9dfz000/4/fffNV5iuXjxIoYMGQIXFxfY29ujR48e2Lt3r8aYtLQ0REZGwtraGm5ubpgxY8Zds2n79u1Dp06dIJfLERAQgNjY2Aa8CiLSBxuOpmHtkTRIJMAXIzrC18VG7JKISI/U6Z9T//vf/wAAn3322V3vSSQSKJXKR6uqjgYPHozmzZtjz549sLKywuLFizF48GD8888/8PDwgFKpRGRkJDw8PHD48GFkZGTghRdegLm5OT788EMAQGpqKiIjIzFhwgSsXbsWu3fvxvjx4+Hp6YmIiAhRrouIGtbJ9AK882v1ovHpA1qgbys3kSsiIn0jEQRBELsIXcjNzYWrqysOHDiAnj17AgCKiopgb2+P+Ph4hIeHY/v27Rg8eDBu3rwJd3d3AMCyZcswc+ZM5OTkwMLCAjNnzkRcXBySk5PVxx4xYgQKCgqwY8eOh6pFm6c5E5F+ySkqxxNfHUKmogwDA92xbFQwpFKueyIyBdr8/K7TLTx91KhRI7Rs2RJr1qxBcXExqqqqsHz5cri5uSE4OBgAkJCQgKCgIHV4AoCIiAgoFAqcPXtWPSY8PFzj2BEREUhISLjnucvLy6FQKDReRGR4KpUqTFp7HJmKMjRztcGiZ9ozPBFRrep0C++999677/tz5sypUzGPQiKR4M8//0RUVBTs7OwglUrh5uaGHTt2wMnJCQCQmZmpEZ4AqL/OzMy87xiFQoHS0lJYWVndde4FCxZg3rx59XFZRNSA5sedR+LVPNjKzbDihRDYWZo/eCciMkl1ClBbtmzR+LqyshKpqakwMzNDs2bNdBqgZs2ahY8//vi+Y86fP4+WLVti0qRJcHNzw8GDB2FlZYWVK1fiiSeewNGjR+Hp6amzmv5r9uzZmDZtmvprhUIBb2/vejsfEenepqTriD18FQDw+bMd0MzVVtyCiEiv1SlAnThx4q5tCoUCMTExGDp06CMX9W/Tp09HTEzMfcf4+/tjz5492Lp1K/Lz89X3Lb/55hvEx8dj9erVmDVrFjw8PJCYmKixb1ZWFgDAw8ND/d+abf8eY29vX+vsEwDI5XLI5fK6XB4R6YEz1wvx5pYzAIDX+jfHgED3B+xBRKZOZ01N7O3tMW/ePDzxxBMYPXq0rg4LV1dXuLo++LEJNQ82lko1l3VJpVKoVCoAQFhYGObPn4/s7Gy4uVV/qiY+Ph729vYIDAxUj9m2bZvGMeLj4xEWFvbI10JE+ufW7XK8/MMxVFSp0L+VG17r31zskojIAOh0EXlhYSEKCwt1eciHFhYWBicnJ0RHR+PUqVO4ePEiZsyYoW5LAAADBw5EYGAgRo8ejVOnTmHnzp14++23MWnSJPUM0oQJE3DlyhW88cYbSElJwTfffIONGzdi6tSpolwXEdWfKqUKk9edwM3CMvi72ODzER24aJyIHkqdZqC+/PJLja8FQUBGRgZ++OEHDBo0SCeFacvFxQU7duzAW2+9hX79+qGyshJt2rTBb7/9hvbt2wMAZDIZtm7diokTJyIsLAw2NjaIjo7WWBTv5+eHuLg4TJ06FV988QW8vLywcuVK9oAiMkIfbU9BwpVbsLGQYfnoYNhz0TgRPaQ69YHy8/PT+FoqlcLV1RX9+vXD7NmzYWdnp7MCDRH7QBHpv99O3sBr608CAJaN6oTH2tbfB02IyDBo8/O7TjNQqampdSqMiEgfnL1ZiJmbTwMAJvVtxvBERFqr0xqosWPHoqio6K7txcXFGDt27CMXRURUX/KLK/DyD0koq1ShdwtXTBvQUuySiMgA1SlArV69GqWlpXdtLy0txZo1ax65KCKi+lClVOGVn07gen4pmjpb48sRHSHjonEiqgOtbuEpFAoIggBBEFBUVARLS0v1e0qlEtu2bVO3ByAi0jcLd13Aocu5sDKXYcULwXCw5qJxIqobrQKUo6MjJBIJJBIJWrRocdf7EomEjzQhIr209fRNLN9/BQCwcHg7tPLgBzyIqO60ClB79+6FIAjo168fNm/eDGdnZ/V7FhYW8PHxQePGjXVeJBHRo0jJVGDGz9WLxl/u7Y/B7fj3FBE9Gq0CVO/evQFUfwqvadOmkEi4doCI9FthSSVe/iEJpZVK9AhwwYyBXDRORI+uTovIfXx8cOjQIYwaNQrdunXDjRs3AAA//PADDh06pNMCiYjqSqkS8NqGE7h2qwReTlb4amRHmMl0+gAGIjJRdfqbZPPmzYiIiICVlRWOHz+O8vJyANWPcvnwww91WiARUV19Hn8R+y7kwNJciuWjg+FkYyF2SURkJOoUoD744AMsW7YM3377LczN//9TLN27d8fx48d1VhwRUV3tSM7Akr2XAQAfP90ObRo7iFwRERmTOgWoCxcuoFevXndtd3BwQEFBwaPWRET0SC5lFWH6xlMAgHE9/DCkQxORKyIiY1OnAOXh4YHLly/ftf3QoUPw9/d/5KKIiOpKUVaJl35IQnGFEl39nTF7UCuxSyIiI1SnAPXiiy/itddew5EjRyCRSHDz5k2sXbsW06dPx8SJE3VdIxHRQ1GpBEzbcBKpucVo7GCJr5/rxEXjRFQv6vQw4VmzZkGlUqF///4oKSlBr169IJfLMWPGDIwfP17XNRIRPVBOUTmW7vsHf57PhoWZFMtHh6CRrVzssojISEkEQRDqunNFRQUuX76M27dvIzAwEMuXL8fChQuRmZmpyxoNjkKhgIODAwoLC2Fvz27HRPVBqRJwMr0A+y9kY++FHJy5Uah+b+Gwdhge4i1idURkiLT5+a3VDFR5eTneffddxMfHq2ecoqKisGrVKgwdOhQymQxTp059pOKJiO7l1u1yHLiUg30XcrD/Yg4KSio13g9q4oCRXZoyPBFRvdMqQM2ZMwfLly9HeHg4Dh8+jOHDh2PMmDH4+++/sWjRIgwfPhwymay+aiUiE6NSCTh9oxD77swynb5egH/PmdtbmqFnC1f0bemGXi1c4GZnee+DERHpkFYB6ueff8aaNWvw5JNPIjk5Ge3atUNVVRVOnTrFx7oQkU7kF1fgwKUc7L8zy3SruELj/UBPe/Rp6Yq+rdzQ0duRi8SJSBRaBajr168jODgYANC2bVvI5XJMnTqV4YmI6kylEnAuQ4G9KdnYdzEHJ9LyofrXLJOt3Aw9m7ugb0s39G7pCnd7zjIRkfi0ClBKpRIWFv//KAQzMzPY2trqvCgiMm6FpZU4dCkXey9kY9+FHOTeLtd4v6W7Hfq0qr41F+zjBHPOMhGRntEqQAmCgJiYGMjl1R8NLisrw4QJE2BjY6Mx7pdfftFdhURk8ARBwPmMIuy7mI19KTlISsuH8l/TTDYWMnQPcEGflm7o09IVjR2tRKyWiOjBtApQ0dHRGl+PGjVKp8UQkfEoKqvEX5dzse9C9afmMhVlGu8HuNmib8vqWaYQX2dYmHGWiYgMh1YBatWqVfVVBxEZOEEQcDHr9p1PzGXj2NV8VP1rlsnKXIZuzRqhTys39GnhCm9naxGrJSJ6NHXqRE5EBADF5VXVs0wXc7AvJRs3CzVnmfxdbNS35br4OcPSnG1OiMg4MEAR0UMTBAH/5BRj353F34mpeahQqtTvy82kCGvWCH3vhCafRjb3ORoRkeFigCKi+yqtUCLhSi72puRg38VspOeVarzf1NkafVu6ok8rN4T5N+IsExGZBAYoIrpLam6xuvv331duoaLq/2eZLGRShPo7q2eZ/Fxs2AuOiEwOAxQRoaxSib+v3LrziblsXL1VovF+E0cr9L3TlymsWSNYW/CvDiIybfxbkMiECYKALSdu4P2t55D/rwfzmssk6OLnjD4t3NC3lSuaudpylomI6F8YoIhM1M2CUry55Qz2XcgBAHjYW6Jvq+rbct0DXGAr518PRET3wr8hiUyMSiVgXWIaPtqegtvlVbAwk2JKeHO82NOfj0whInpIDFBEJuRqbjFmbj6NI6l5AIBgHyd8/HQ7BLjxmZZERNpggCIyAUqVgO8PpWJR/AWUVapgbSHDGxEtMTrMFzIp1zYREWmLAYrIyF3ILMIbm0/jVHoBAKBHgAsWPBXER6kQET0CBigiI1VRpcLSff9gyd5LqFQKsLM0w9uRrfFMiDc/UUdE9IgYoIiM0OnrBXhj02mkZBYBAMJbu2P+0LZwt7cUuTIiIuPAAEVkRMoqlfj8z4v49sAVqATA2cYC855sg8HtPDnrRESkQwxQREYiMTUPMzefRmpuMQDgyfaNMfeJQDSylYtcGRGR8TGYpi/z589Ht27dYG1tDUdHx1rHpKWlITIyEtbW1nBzc8OMGTNQVVWlMWbfvn3o1KkT5HI5AgICEBsbe9dxvv76a/j6+sLS0hKhoaFITEyshysi0o3b5VWY81synlmegNTcYrjby7HyhRB8ObIjwxMRUT0xmABVUVGB4cOHY+LEibW+r1QqERkZiYqKChw+fBirV69GbGws5syZox6TmpqKyMhI9O3bFydPnsSUKVMwfvx47Ny5Uz1mw4YNmDZtGubOnYvjx4+jffv2iIiIQHZ2dr1fI5G2DlzMQcTnB7Am4RoAYERnb+ya2hvhge4iV0ZEZNwkgiAIYhehjdjYWEyZMgUFBQUa27dv347Bgwfj5s2bcHev/uGxbNkyzJw5Ezk5ObCwsMDMmTMRFxeH5ORk9X4jRoxAQUEBduzYAQAIDQ1F586dsWTJEgCASqWCt7c3XnnlFcyaNeuhalQoFHBwcEBhYSHs7e11cNVEmgpLKvF+3DlsSroOAPB2tsJHT7VD9wAXkSsjIjJc2vz8NpgZqAdJSEhAUFCQOjwBQEREBBQKBc6ePaseEx4errFfREQEEhISAFTPciUlJWmMkUqlCA8PV4+pTXl5ORQKhcaLqL7sSM5E+Of7sSnpOiQSYEx3X+yc0ovhiYioARnNIvLMzEyN8ARA/XVmZuZ9xygUCpSWliI/Px9KpbLWMSkpKfc894IFCzBv3jxdXAbRPeUUlePd388i7kwGAKCZqw0+GdYOwT7OIldGRGR6RJ2BmjVrFiQSyX1f9wsu+mL27NkoLCxUv9LT08UuiYyIIAjYcuI6Bny+H3FnMiCTSjCpbzPEvdqT4YmISCSizkBNnz4dMTEx9x3j7+//UMfy8PC469NyWVlZ6vdq/luz7d9j7O3tYWVlBZlMBplMVuuYmmPURi6XQy7np51I924WlOKtLWew90IOACDQ0x6fDGuHtk0cRK6MiMi0iRqgXF1d4erqqpNjhYWFYf78+cjOzoabmxsAID4+Hvb29ggMDFSP2bZtm8Z+8fHxCAsLAwBYWFggODgYu3fvRlRUFIDqReS7d+/G5MmTdVIn0cNQqQT8dDQNC7al4HZ5FSxkUrwW3hwv9fKHucxoli4SERksg1kDlZaWhry8PKSlpUGpVOLkyZMAgICAANja2mLgwIEIDAzE6NGj8cknnyAzMxNvv/02Jk2apJ4dmjBhApYsWYI33ngDY8eOxZ49e7Bx40bExcWpzzNt2jRER0cjJCQEXbp0weLFi1FcXIwxY8aIcdlkgq7mFmPWL6fx95U8AECnpo74ZFg7BLjZiVwZERHVMJg2BjExMVi9evVd2/fu3Ys+ffoAAK5du4aJEydi3759sLGxQXR0ND766COYmf1/Tty3bx+mTp2Kc+fOwcvLC++8885dtxGXLFmChQsXIjMzEx06dMCXX36J0NDQh66VbQyoLpQqAd8fSsWi+Asoq1TBylyGGREtEd3NFzIpH8NCRFTftPn5bTABypAwQN2fUiXgt5M34GhtjmAfZzhYmYtdkuguZhXhjU2ncTK9AADQPaARFgxth6aNrMUtjIjIhGjz89tgbuGR8fgh4Sre/eMcAEAiAVp72KOLnzO6+Dmjs68zXO1MZ0F+RZUKy/b/g6/2XEKlUoCd3AxvRbbGs529+fBfIiI9xgBFDUqpErDyUCoAwMVWjtzb5TiXocC5DAViD18FAPi72iD0Tpjq4ucMLyfjnIU5c70QMzadQkpmEQAgvLUbPogKgoeDpciVERHRgzBAUYPadTYT1/NL4WRtjoNv9EVReSWOpuYjMfUWjqTm4UJWEa7kFONKTjF+Sqzup9XE0UpjhqqZq41Bz86UVSqx+M9L+PbgFShVApxtLDD3iUA82b6xQV8XEZEpYYCiBlUz+zSqqw+sLGSwspAhsp0nItt5Aqh+xtuxa3lITM3DkdQ8JN8oxI2CUmw5cQNbTtwAADSysdAIVK097Q1mkfXRq3mYuek0ruQWAwCeaN8Y7z4RiEa2pnPbkojIGDBAUYM5npaPpGv5sJBJMTrMp9YxDtbm6N/aHf1bVz9Op6SiCifSCnAkNQ+JqbdwIq0At4orsD05E9uTqx/RYyc3Q4ivE7r4NUIXPycENXGEhZl+9Uq6XV6FhTtSsObvaxAEwM1OjvlDgzAg0P3BOxMRkd5hgKIG893B6tmnJzs0hpvdw63zsbYwQ/cAF/WDcsurlDhzvRCJV6tnqY5dzUdReRX2XshRd+u2NJeio7eTepaqY1NHWFuI91v9wMUczP7lDG4UlAIAng3xxpuRrfnpQyIiA8Y2BvWAbQzulp5Xgt4L90IlANtf64nWnrr5dVGqBJzPUOBIah6OpuYh8Woe8oorNMaYSSUI8nKoDlS+zgjxcYaDdf2Hl8KSSrwfdw6bkq4DALycrPDRU+3Qo7lLvZ+biIi0xzYGpHdiD1+FSgB6NnfRWXgCAJlUgrZNHNC2iQPG9fCDIAj4J+e2OlAdSc1DRmEZTqQV4ERaAZbvvwKJBGjlYa/+pF9nP6eHnhF7WDuSM/HOb8nIKSqHRAJEh/liRkRL2Mj5R46IyBhwBqoecAZKk6KsEt0W7MHt8irEjumMPi3dGuzcgiDgen4pElPzcPTObb+aBdz/5u9io16UXt06wapOn4jLvV2Oub+fRdzpjOrjutrgk6fbIcTX+ZGvhYiI6hdnoEivbDyajtvlVWjuZoveLXTz8OiHJZFI4O1sDW9nazwd7AUAyC4qU7dOSLyaj5RMBa7kFuNKbjHWH61undDYwbI6UPk5I9TPGc1cbe8bqARBwG8nb2LeH2eRX1IJmVSCl3v549X+zWFpLmuQayUioobDGah6wBmo/1elVKH3wn24UVCKj54KwoguTcUu6S7/bp2QeDUPZ64Xokql+ceikY3Fndt91YHq360TMgpL8daWZOxJyQYAtPa0x8Jh7dC2iUODXwsREdUdZ6BIb2xPzsSNglI0srFAVMcmYpdTq4dtnbDjbCZ2nP3/1gnBvk4IcLXFhqPpKCqvgoVMilf7B+Dl3s1gLtOvNgpERKRbDFBUbwRBwMqDVwBUN840lFtZtbVOSL5ReCdQ/X/rhH0XcrDvTuuEDt6OWDisHZq724lZOhERNRAGKKo3Sdfycep6ISzM7t040xDIzWQI9nFGsI8z/tfn/1snJN7plN6xqSOeC/UxmG7oRET06BigqN6svNM486mOTeBiRI8q+XfrBCIiMk1cqEH14tqtYuw8V71eaGwPP5GrISIi0i0GKKoXq/66CkEAerdwRQuuCyIiIiPDAEU6V1hSiY3Hqvspje/J2SciIjI+DFCkcz8dTUNJhRKtPOzQI4DPfSMiIuPDAEU6ValUIfavqwCAcT386vQ4FCIiIn3HAEU6te1MBjIVZXCxlePJDo3FLoeIiKheMECRzgiCgG/vNM6MDvOB3MwwGmcSERFpiwGKdKa6saQCcjMpnu9quI0ziYiIHoQBinTm2zuNM58O9oKzjYXI1RAREdUfBijSiSs5t7E7JQsAMLY7WxcQEZFxY4AinahpnNm/lRsC3GzFLoeIiKheMUDRIysoqcDPSdWNM8excSYREZkABih6ZGuPpKGsUoVAT3uE+TcSuxwiIqJ6xwBFj6SiSoXVh68CqH5sCxtnEhGRKWCAokfyx6mbyC4qh5udHIPbsXEmERGZBgYoqjNBELDyUHXrguhuvrAw428nIiIyDfyJR3WW8M8tnM9QwMpchudDm4pdDhERUYNhgKI6q5l9Gh7iBUdrNs4kIiLTwQBFdXI5+zb2pGRDIgHGsHEmERGZGAYoqpPv/6qefQpv7Q4/FxuRqyEiImpYDFCktVu3y7E56ToAYHwPzj4REZHpYYAira09kobyKhWCmjigi5+z2OUQERE1OAYo0kpZpRJrEq4CYONMIiIyXQxQpJXfT91E7u0KeDpY4vEgT7HLISIiEoXBBKj58+ejW7dusLa2hqOj413vnzp1CiNHjoS3tzesrKzQunVrfPHFF3eN27dvHzp16gS5XI6AgADExsbeNebrr7+Gr68vLC0tERoaisTExHq4IsMjCAK+O1i9eDymmy/MZQbz24eIiEinDOYnYEVFBYYPH46JEyfW+n5SUhLc3Nzw448/4uzZs3jrrbcwe/ZsLFmyRD0mNTUVkZGR6Nu3L06ePIkpU6Zg/Pjx2Llzp3rMhg0bMG3aNMydOxfHjx9H+/btERERgezs7Hq/Rn136HIuLmQVwdpChhFd2DiTiIhMl0QQBEHsIrQRGxuLKVOmoKCg4IFjJ02ahPPnz2PPnj0AgJkzZyIuLg7JycnqMSNGjEBBQQF27NgBAAgNDUXnzp3VwUulUsHb2xuvvPIKZs2a9VA1KhQKODg4oLCwEPb29lpeof564ftEHLiYg5huvnj3yTZil0NERKRT2vz8NpgZqLooLCyEs/P/f0osISEB4eHhGmMiIiKQkJAAoHqWKykpSWOMVCpFeHi4ekxtysvLoVAoNF7G5kJmEQ5czIFUAoxl40wiIjJxRhugDh8+jA0bNuCll15Sb8vMzIS7u7vGOHd3dygUCpSWliI3NxdKpbLWMZmZmfc814IFC+Dg4KB+eXt76/Zi9MD3dx7bEtHGA00bWYtcDRERkbhEDVCzZs2CRCK57yslJUXr4yYnJ2PIkCGYO3cuBg4cWA+Va5o9ezYKCwvVr/T09Ho/Z0PKKSrHlpM3AFS3LiAiIjJ1ZmKefPr06YiJibnvGH9/f62Oee7cOfTv3x8vvfQS3n77bY33PDw8kJWVpbEtKysL9vb2sLKygkwmg0wmq3WMh4fHPc8pl8shl8u1qtOQ/Pj3NVRUqdDB2xGdmjqJXQ4REZHoRA1Qrq6ucHV11dnxzp49i379+iE6Ohrz58+/6/2wsDBs27ZNY1t8fDzCwsIAABYWFggODsbu3bsRFRUFoHoR+e7duzF58mSd1WlIyiqV+PHvawDYOJOIiKiGqAFKG2lpacjLy0NaWhqUSiVOnjwJAAgICICtrS2Sk5PRr18/REREYNq0aeo1SzKZTB3SJkyYgCVLluCNN97A2LFjsWfPHmzcuBFxcXHq80ybNg3R0dEICQlBly5dsHjxYhQXF2PMmDENfs36YMuJG7hVXIEmjlZ4rM29Z+GIiIhMicEEqDlz5mD16tXqrzt27AgA2Lt3L/r06YNNmzYhJycHP/74I3788Uf1OB8fH1y9ehUA4Ofnh7i4OEydOhVffPEFvLy8sHLlSkRERKjHP/vss8jJycGcOXOQmZmJDh06YMeOHXctLDcFKpWA7+4sHh/T3RdmbJxJREQEwAD7QBkCY+kDtfdCNsasOgpbuRkSZveDnaW52CURERHVG/aBIp2oeWzLiM7eDE9ERET/wgBFtTqfocChy7mQSoCY7r5il0NERKRXGKCoVjVrnwYFecLLiY0ziYiI/o0Biu6SrSjDbzWNM3uwcSYREdF/MUDRXdYkXEOlUkCwjxM6snEmERHRXRigSENphRI/HqlunPkiH9tCRERUKwYo0rD5+HUUlFTC29kKAwLZOJOIiKg2DFCkplIJ+P7O4vGx3f0gk/KxLURERLVhgCK1vReycSW3GHaWZhge4i12OURERHqLAYrUvj14BQDwXJemsJUbzFN+iIiIGhwDFAEAkm8U4u8reTCTShDdzVfscoiIiPQaAxQB+P/GmZHtPNHY0UrkaoiIiPQbAxQhs7AMf5y6CQAYx8aZRERED8QARVidcBVVKgFd/JzRzstR7HKIiIj0HgOUiSsur8Lav6sbZ/KxLURERA+HAcrEbT5+HYqyKvg2skb/1u5il0NERGQQGKBMmFIlqBePj+3BxplEREQPiwHKhP15PgvXbpXAwcocw4K9xC6HiIjIYDBAmbDvDlbPPj0f2hTWFmycSURE9LAYoEzUqfQCJF7Ng7mMjTOJiIi0xQBlomrWPj3RrjHc7S1FroaIiMiwMECZoJsFpYg7kwEAGNeTrQuIiIi0xQBlgmIPX4VSJaBbs0Zo09hB7HKIiIgMDgOUibldXoWfjqQBAMZz9omIiKhOGKBMzMaj6Sgqr4K/qw36tHATuxwiIiKDxABlQpQqAd//Vb14fFwPP0jZOJOIiKhOGKBMyK6zmbieXwona3M81ZGNM4mIiOqKAcqErLzTumBUVx9YWchEroaIiMhwMUCZiONp+Ui6lg8LmRSjw3zELoeIiMigMUCZiJrHtgzp0BhudmycSURE9CgYoExAel4JtiezcSYREZGuMECZgNjDV6ESgJ7NXdDKw17scoiIiAweA5SRU5RVYsPRdADVrQuIiIjo0TFAGbmNR9Nxu7wKzd1s0buFq9jlEBERGQUGKCNWpVRh1V9XAVQ/tkUiYeNMIiIiXWCAMmLbkzNxo6AUjWwsMKRDE7HLISIiMhoMUEZKEASsPHgFADA6zAeW5mycSUREpCsMUEYq6Vo+Tl0vhIWZFKO6snEmERGRLhlMgJo/fz66desGa2trODo63nfsrVu34OXlBYlEgoKCAo339u3bh06dOkEulyMgIACxsbF37f/111/D19cXlpaWCA0NRWJiou4upIGsvNM486mOTeBiKxe5GiIiIuNiMAGqoqICw4cPx8SJEx84dty4cWjXrt1d21NTUxEZGYm+ffvi5MmTmDJlCsaPH4+dO3eqx2zYsAHTpk3D3Llzcfz4cbRv3x4RERHIzs7W6fXUp2u3irHzXCYAti4gIiKqDwYToObNm4epU6ciKCjovuOWLl2KgoICvP7663e9t2zZMvj5+WHRokVo3bo1Jk+ejGHDhuHzzz9Xj/nss8/w4osvYsyYMQgMDMSyZctgbW2N77//XufXVF9W/XUVggD0aemK5u52YpdDRERkdAwmQD2Mc+fO4b333sOaNWsgld59aQkJCQgPD9fYFhERgYSEBADVs1xJSUkaY6RSKcLDw9VjalNeXg6FQqHxEkthSSU2HqtunDm+h79odRARERkzowlQ5eXlGDlyJBYuXIimTZvWOiYzMxPu7u4a29zd3aFQKFBaWorc3Fwolcpax2RmZt7z3AsWLICDg4P65e3t/egXVEc/HU1DSYUSrTzs0D2gkWh1EBERGTNRA9SsWbMgkUju+0pJSXmoY82ePRutW7fGqFGj6rnq2s9dWFiofqWnpzd4DQBQqVQh9k7jzHE92DiTiIiovpiJefLp06cjJibmvmP8/R/uNtSePXtw5swZbNq0CUB1HyQAcHFxwVtvvYV58+bBw8MDWVlZGvtlZWXB3t4eVlZWkMlkkMlktY7x8PC457nlcjnkcvE/6bbtTAYyFWVwsZXjyQ6NxS6HiIjIaIkaoFxdXeHqqpvns23evBmlpaXqr48ePYqxY8fi4MGDaNasGQAgLCwM27Zt09gvPj4eYWFhAAALCwsEBwdj9+7diIqKAgCoVCrs3r0bkydP1kmd9UUQBHx7p3FmdJgP5GZsnElERFRfRA1Q2khLS0NeXh7S0tKgVCpx8uRJAEBAQABsbW3VIalGbm4uAKB169bqvlETJkzAkiVL8MYbb2Ds2LHYs2cPNm7ciLi4OPV+06ZNQ3R0NEJCQtClSxcsXrwYxcXFGDNmTINcZ10dSc1D8g0FLM2leJ6NM4mIiOqVwQSoOXPmYPXq1eqvO3bsCADYu3cv+vTp81DH8PPzQ1xcHKZOnYovvvgCXl5eWLlyJSIiItRjnn32WeTk5GDOnDnIzMxEhw4dsGPHjrsWluubmsaZT3fygrONhcjVEBERGTeJULNYiHRGoVDAwcEBhYWFsLe3r/fzXcm5jf6f7YcgALun90YzV9t6PycREZGx0ebnt9G0MTBlNY0z+7dyY3giIiJqAAxQBq6gpAI/J1W3TRjXk49tISIiaggMUAZu7ZE0lFWqEOhpjzB/Ns4kIiJqCAxQBqyiSoXVh68CAF7sxcaZREREDYUByoD9ceomsovK4W4vR2QQG2cSERE1FAYoAyUIAlYeqm5dEN3NFxZm/FYSERE1FP7UNVAJ/9zC+QwFrMxleK5L7Q9PJiIiovrBAGWgamafhod4wdGajTOJiIgaEgOUAbqcfRt7UrIhkQBju7N1ARERUUNjgDJA392ZfRrQ2h2+LjYiV0NERGR6GKAMzK3b5fjl+HUAwPie/iJXQ0REZJoYoAzM2iNpKK9SoZ2XAzr7OoldDhERkUligDIgZZVKrEm4CgAY14ONM4mIiMTCAGVAfj91E7m3K+DpYInHgzzFLoeIiMhkMUAZkLziCliaSxHTzRfmMn7riIiIxGImdgH08Cb0boZnQ7xhzq7jREREomKAMjBONmyaSUREJDZOZRARERFpiQGKiIiISEsMUERERERaYoAiIiIi0hIDFBEREZGWGKCIiIiItMQARURERKQlBigiIiIiLTFAEREREWmJAYqIiIhISwxQRERERFpigCIiIiLSEgMUERERkZbMxC7AGAmCAABQKBQiV0JEREQPq+bnds3P8fthgKoHRUVFAABvb2+RKyEiIiJtFRUVwcHB4b5jJMLDxCzSikqlws2bN2FnZweJRKLTYysUCnh7eyM9PR329vY6PTZpj98P/cLvh37h90O/8PvxYIIgoKioCI0bN4ZUev9VTpyBqgdSqRReXl71eg57e3v+AdAj/H7oF34/9Au/H/qF34/7e9DMUw0uIiciIiLSEgMUERERkZYYoAyMXC7H3LlzIZfLxS6FwO+HvuH3Q7/w+6Ff+P3QLS4iJyIiItISZ6CIiIiItMQARURERKQlBigiIiIiLTFAEREREWmJAcqAfP311/D19YWlpSVCQ0ORmJgodkkmacGCBejcuTPs7Ozg5uaGqKgoXLhwQeyy6I6PPvoIEokEU6ZMEbsUk3bjxg2MGjUKjRo1gpWVFYKCgnDs2DGxyzJJSqUS77zzDvz8/GBlZYVmzZrh/ffff6jnvdG9MUAZiA0bNmDatGmYO3cujh8/jvbt2yMiIgLZ2dlil2Zy9u/fj0mTJuHvv/9GfHw8KisrMXDgQBQXF4tdmsk7evQoli9fjnbt2oldiknLz89H9+7dYW5uju3bt+PcuXNYtGgRnJycxC7NJH388cdYunQplixZgvPnz+Pjjz/GJ598gq+++krs0gwa2xgYiNDQUHTu3BlLliwBUP28PW9vb7zyyiuYNWuWyNWZtpycHLi5uWH//v3o1auX2OWYrNu3b6NTp0745ptv8MEHH6BDhw5YvHix2GWZpFmzZuGvv/7CwYMHxS6FAAwePBju7u747rvv1NuefvppWFlZ4ccffxSxMsPGGSgDUFFRgaSkJISHh6u3SaVShIeHIyEhQcTKCAAKCwsBAM7OziJXYtomTZqEyMhIjT8nJI7ff/8dISEhGD58ONzc3NCxY0d8++23Ypdlsrp164bdu3fj4sWLAIBTp07h0KFDGDRokMiVGTY+TNgA5ObmQqlUwt3dXWO7u7s7UlJSRKqKgOqZwClTpqB79+5o27at2OWYrPXr1+P48eM4evSo2KUQgCtXrmDp0qWYNm0a3nzzTRw9ehSvvvoqLCwsEB0dLXZ5JmfWrFlQKBRo1aoVZDIZlEol5s+fj+eff17s0gwaAxTRI5g0aRKSk5Nx6NAhsUsxWenp6XjttdcQHx8PS0tLscshVP/DIiQkBB9++CEAoGPHjkhOTsayZcsYoESwceNGrF27FuvWrUObNm1w8uRJTJkyBY0bN+b34xEwQBkAFxcXyGQyZGVlaWzPysqCh4eHSFXR5MmTsXXrVhw4cABeXl5il2OykpKSkJ2djU6dOqm3KZVKHDhwAEuWLEF5eTlkMpmIFZoeT09PBAYGamxr3bo1Nm/eLFJFpm3GjBmYNWsWRowYAQAICgrCtWvXsGDBAgaoR8A1UAbAwsICwcHB2L17t3qbSqXC7t27ERYWJmJlpkkQBEyePBlbtmzBnj174OfnJ3ZJJq1///44c+YMTp48qX6FhITg+eefx8mTJxmeRNC9e/e7WntcvHgRPj4+IlVk2kpKSiCVav64l8lkUKlUIlVkHDgDZSCmTZuG6OhohISEoEuXLli8eDGKi4sxZswYsUszOZMmTcK6devw22+/wc7ODpmZmQAABwcHWFlZiVyd6bGzs7tr/ZmNjQ0aNWrEdWkimTp1Krp164YPP/wQzzzzDBITE7FixQqsWLFC7NJM0hNPPIH58+ejadOmaNOmDU6cOIHPPvsMY8eOFbs0g8Y2BgZkyZIlWLhwITIzM9GhQwd8+eWXCA0NFbsskyORSGrdvmrVKsTExDRsMVSrPn36sI2ByLZu3YrZs2fj0qVL8PPzw7Rp0/Diiy+KXZZJKioqwjvvvIMtW7YgOzsbjRs3xsiRIzFnzhxYWFiIXZ7BYoAiIiIi0hLXQBERERFpiQGKiIiISEsMUERERERaYoAiIiIi0hIDFBEREZGWGKCIiIiItMQARURERKQlBigiMmlXr16FRCLByZMn6+0cMTExiIqKqrfjE1HDY4AiIoMWExMDiURy1+uxxx57qP29vb2RkZHBx74QkVb4LDwiMniPPfYYVq1apbFNLpc/1L4ymQweHh71URYRGTHOQBGRwZPL5fDw8NB4OTk5Aah+duHSpUsxaNAgWFlZwd/fH5s2bVLv+99bePn5+Xj++efh6uoKKysrNG/eXCOcnTlzBv369YOVlRUaNWqEl156Cbdv31a/r1QqMW3aNDg6OqJRo0Z444038N8nZqlUKixYsAB+fn6wsrJC+/btNWp6UA1EJD4GKCIyeu+88w6efvppnDp1Cs8//zxGjBiB8+fP33PsuXPnsH37dpw/fx5Lly6Fi4sLAKC4uBgRERFwcnLC0aNH8fPPP+PPP//E5MmT1fsvWrQIsbGx+P7773Ho0CHk5eVhy5YtGudYsGAB1qxZg2XLluHs2bOYOnUqRo0ahf379z+wBiLSEwIRkQGLjo4WZDKZYGNjo/GaP3++IAiCAECYMGGCxj6hoaHCxIkTBUEQhNTUVAGAcOLECUEQBOGJJ54QxowZU+u5VqxYITg5OQm3b99Wb4uLixOkUqmQmZkpCIIgeHp6Cp988on6/crKSsHLy0sYMmSIIAiCUFZWJlhbWwuHDx/WOPa4ceOEkSNHPrAGItIPXANFRAavb9++WLp0qcY2Z2dn9f+HhYVpvBcWFnbPT91NnDgRTz/9NI4fP46BAwciKioK3bp1AwCcP38e7du3h42NjXp89+7doVKpcOHCBVhaWiIjIwOhoaHq983MzBASEqK+jXf58mWUlJRgwIABGuetqKhAx44dH1gDEekHBigiMng2NjYICAjQybEGDRqEa9euYdu2bYiPj0f//v0xadIkfPrppzo5fs16qbi4ODRp0kTjvZqF7/VdAxE9Oq6BIiKj9/fff9/1devWre853tXVFdHR0fjxxx+xePFirFixAgDQunVrnDp1CsXFxeqxf/31F6RSKVq2bAkHBwd4enriyJEj6verqqqQlJSk/jowMBByuRxpaWkICAjQeHl7ez+wBiLSD5yBIiKDV15ejszMTI1tZmZm6oXXP//8M0JCQtCjRw+sXbsWiYmJ+O6772o91pw5cxAcHIw2bdqgvLwcW7duVYet559/HnPnzkV0dDTeffdd5OTk4JVXXsHo0aPh7u4OAHjttdfw0UcfoXnz5mjVqhU+++wzFBQUqI9vZ2eH119/HVOnToVKpUKPHj1QWFiIv/76C/b29oiOjr5vDUSkHxigiMjg7dixA56enhrbWrZsiZSUFADAvHnzsH79evzvf/+Dp6cnfvrpJwQGBtZ6LAsLC8yePRtXr16FlZUVevbsifXr1wMArK2tsXPnTrz22mvo3LkzrK2t8fTTT+Ozzz5T7z99+nRkZGQgOjoaUqkUY8eOxdChQ1FYWKge8/7778PV1RULFizAlStX4OjoiE6dOuHNN998YA1EpB8kgvCfBiVEREZEIpFgy5YtfJQKEekU10ARERERaYkBioiIiEhLXANFREaNqxSIqD5wBoqIiIhISwxQRERERFpigCIiIiLSEgMUERERkZYYoIiIiIi0xABFREREpCUGKCIiIiItMUARERERaYkBioiIiEhL/weTw4W2q/Nb8wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "buffer_size = 10**5\n",
    "n_sequence = 50\n",
    "elite_ratio = 0.2\n",
    "plan_horizon = 25\n",
    "num_episodes = 10\n",
    "env_name = 'Pendulum-v1'\n",
    "env = gym.make(env_name)\n",
    "\n",
    "replaybuffer = ReplayBuffer(buffer_size)\n",
    "pets = PETS(env, replaybuffer, n_sequence, elite_ratio, plan_horizon, num_episodes)\n",
    "return_list = pets.train()\n",
    "\n",
    "episodes_list = list(range(len(return_list)))\n",
    "plt.plot(episodes_list, return_list)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('PETS on {}'.format(env_name))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
